diff --git a/Pipfile b/Pipfile index 9552fc7a..ddd9bc3b 100644 --- a/Pipfile +++ b/Pipfile @@ -22,9 +22,11 @@ tox = "*" detox = "*" httpbin = "==0.5.0" pytest-mypy = "*" +black = {git = "https://github.com/ambv/black.git", editable = true} +"e1839a8" = {path = ".", editable = true, extras=["socks"]} +mypy = "==0.540" +win-inet-ptonsocks = {version="*", os_name = "=='windows'"} [packages] -"e1839a8" = {path = ".", editable = true, extras=["socks"]} -mypy = "==0.540" diff --git a/docs/_themes/flask_theme_support.py b/docs/_themes/flask_theme_support.py index 33f47449..814e8da2 100644 --- a/docs/_themes/flask_theme_support.py +++ b/docs/_themes/flask_theme_support.py @@ -1,86 +1,76 @@ # flasky extensions. flasky pygments style based on tango style from pygments.style import Style -from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic, Whitespace, Punctuation, Other, Literal +from pygments.token import Keyword, Name, Comment, String, Error, Number, Operator, Generic, Whitespace, Punctuation, Other, Literal class FlaskyStyle(Style): background_color = "#f8f8f8" default_style = "" - styles = { # No corresponding class for the following: #Text: "", # class: '' - Whitespace: "underline #f8f8f8", # class: 'w' - Error: "#a40000 border:#ef2929", # class: 'err' - Other: "#000000", # class 'x' - - Comment: "italic #8f5902", # class: 'c' - Comment.Preproc: "noitalic", # class: 'cp' - - Keyword: "bold #004461", # class: 'k' - Keyword.Constant: "bold #004461", # class: 'kc' - Keyword.Declaration: "bold #004461", # class: 'kd' - Keyword.Namespace: "bold #004461", # class: 'kn' - Keyword.Pseudo: "bold #004461", # class: 'kp' - Keyword.Reserved: "bold #004461", # class: 'kr' - Keyword.Type: "bold #004461", # class: 'kt' - - Operator: "#582800", # class: 'o' - Operator.Word: "bold #004461", # class: 'ow' - like keywords - - Punctuation: "bold #000000", # class: 'p' - + Whitespace: "underline #f8f8f8", # class: 'w' + Error: "#a40000 border:#ef2929", # class: 'err' + Other: "#000000", # class 'x' + Comment: "italic #8f5902", # class: 'c' + Comment.Preproc: "noitalic", # class: 'cp' + Keyword: "bold #004461", # class: 'k' + Keyword.Constant: "bold #004461", # class: 'kc' + Keyword.Declaration: "bold #004461", # class: 'kd' + Keyword.Namespace: "bold #004461", # class: 'kn' + Keyword.Pseudo: "bold #004461", # class: 'kp' + Keyword.Reserved: "bold #004461", # class: 'kr' + Keyword.Type: "bold #004461", # class: 'kt' + Operator: "#582800", # class: 'o' + Operator.Word: "bold #004461", # class: 'ow' - like keywords + Punctuation: "bold #000000", + # class: 'p' # because special names such as Name.Class, Name.Function, etc. # are not recognized as such later in the parsing, we choose them # to look the same as ordinary variables. - Name: "#000000", # class: 'n' - Name.Attribute: "#c4a000", # class: 'na' - to be revised - Name.Builtin: "#004461", # class: 'nb' - Name.Builtin.Pseudo: "#3465a4", # class: 'bp' - Name.Class: "#000000", # class: 'nc' - to be revised - Name.Constant: "#000000", # class: 'no' - to be revised - Name.Decorator: "#888", # class: 'nd' - to be revised - Name.Entity: "#ce5c00", # class: 'ni' - Name.Exception: "bold #cc0000", # class: 'ne' - Name.Function: "#000000", # class: 'nf' - Name.Property: "#000000", # class: 'py' - Name.Label: "#f57900", # class: 'nl' - Name.Namespace: "#000000", # class: 'nn' - to be revised - Name.Other: "#000000", # class: 'nx' - Name.Tag: "bold #004461", # class: 'nt' - like a keyword - Name.Variable: "#000000", # class: 'nv' - to be revised - Name.Variable.Class: "#000000", # class: 'vc' - to be revised - Name.Variable.Global: "#000000", # class: 'vg' - to be revised - Name.Variable.Instance: "#000000", # class: 'vi' - to be revised - - Number: "#990000", # class: 'm' - - Literal: "#000000", # class: 'l' - Literal.Date: "#000000", # class: 'ld' - - String: "#4e9a06", # class: 's' - String.Backtick: "#4e9a06", # class: 'sb' - String.Char: "#4e9a06", # class: 'sc' - String.Doc: "italic #8f5902", # class: 'sd' - like a comment - String.Double: "#4e9a06", # class: 's2' - String.Escape: "#4e9a06", # class: 'se' - String.Heredoc: "#4e9a06", # class: 'sh' - String.Interpol: "#4e9a06", # class: 'si' - String.Other: "#4e9a06", # class: 'sx' - String.Regex: "#4e9a06", # class: 'sr' - String.Single: "#4e9a06", # class: 's1' - String.Symbol: "#4e9a06", # class: 'ss' - - Generic: "#000000", # class: 'g' - Generic.Deleted: "#a40000", # class: 'gd' - Generic.Emph: "italic #000000", # class: 'ge' - Generic.Error: "#ef2929", # class: 'gr' - Generic.Heading: "bold #000080", # class: 'gh' - Generic.Inserted: "#00A000", # class: 'gi' - Generic.Output: "#888", # class: 'go' - Generic.Prompt: "#745334", # class: 'gp' - Generic.Strong: "bold #000000", # class: 'gs' - Generic.Subheading: "bold #800080", # class: 'gu' - Generic.Traceback: "bold #a40000", # class: 'gt' + Name: "#000000", # class: 'n' + Name.Attribute: "#c4a000", # class: 'na' - to be revised + Name.Builtin: "#004461", # class: 'nb' + Name.Builtin.Pseudo: "#3465a4", # class: 'bp' + Name.Class: "#000000", # class: 'nc' - to be revised + Name.Constant: "#000000", # class: 'no' - to be revised + Name.Decorator: "#888", # class: 'nd' - to be revised + Name.Entity: "#ce5c00", # class: 'ni' + Name.Exception: "bold #cc0000", # class: 'ne' + Name.Function: "#000000", # class: 'nf' + Name.Property: "#000000", # class: 'py' + Name.Label: "#f57900", # class: 'nl' + Name.Namespace: "#000000", # class: 'nn' - to be revised + Name.Other: "#000000", # class: 'nx' + Name.Tag: "bold #004461", # class: 'nt' - like a keyword + Name.Variable: "#000000", # class: 'nv' - to be revised + Name.Variable.Class: "#000000", # class: 'vc' - to be revised + Name.Variable.Global: "#000000", # class: 'vg' - to be revised + Name.Variable.Instance: "#000000", # class: 'vi' - to be revised + Number: "#990000", # class: 'm' + Literal: "#000000", # class: 'l' + Literal.Date: "#000000", # class: 'ld' + String: "#4e9a06", # class: 's' + String.Backtick: "#4e9a06", # class: 'sb' + String.Char: "#4e9a06", # class: 'sc' + String.Doc: "italic #8f5902", # class: 'sd' - like a comment + String.Double: "#4e9a06", # class: 's2' + String.Escape: "#4e9a06", # class: 'se' + String.Heredoc: "#4e9a06", # class: 'sh' + String.Interpol: "#4e9a06", # class: 'si' + String.Other: "#4e9a06", # class: 'sx' + String.Regex: "#4e9a06", # class: 'sr' + String.Single: "#4e9a06", # class: 's1' + String.Symbol: "#4e9a06", # class: 'ss' + Generic: "#000000", # class: 'g' + Generic.Deleted: "#a40000", # class: 'gd' + Generic.Emph: "italic #000000", # class: 'ge' + Generic.Error: "#ef2929", # class: 'gr' + Generic.Heading: "bold #000080", # class: 'gh' + Generic.Inserted: "#00A000", # class: 'gi' + Generic.Output: "#888", # class: 'go' + Generic.Prompt: "#745334", # class: 'gp' + Generic.Strong: "bold #000000", # class: 'gs' + Generic.Subheading: "bold #800080", # class: 'gu' + Generic.Traceback: "bold #a40000", # class: 'gt' } diff --git a/requests/__init__.py b/requests/__init__.py index bd9c8bb3..d638c667 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -1,10 +1,8 @@ # -*- coding: utf-8 -*- - # __ # /__) _ _ _ _ _/ _ # / ( (- (/ (/ (- _) / _) # / - """ Requests HTTP Library ~~~~~~~~~~~~~~~~~~~~~ @@ -49,11 +47,9 @@ from .exceptions import RequestsDependencyWarning def check_compatibility(urllib3_version, chardet_version): urllib3_version = urllib3_version.split('.') assert urllib3_version != ['dev'] # Verify urllib3 isn't installed from git. - # Sometimes, urllib3 only reports its version as 16.1. if len(urllib3_version) == 2: urllib3_version.append('0') - # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) @@ -61,7 +57,6 @@ def check_compatibility(urllib3_version, chardet_version): assert major == 1 assert minor >= 21 assert minor <= 22 - # Check chardet for compatibility. major, minor, patch = chardet_version.split('.')[:3] major, minor, patch = int(major), int(minor), int(patch) @@ -79,45 +74,56 @@ def _check_cryptography(cryptography_version): return if cryptography_version < [1, 3, 4]: - warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version) + warning = 'Old version of cryptography ({0}) may cause slowdown.'.format( + cryptography_version + ) warnings.warn(warning, RequestsDependencyWarning) + # Check imported dependencies for compatibility. try: check_compatibility(urllib3.__version__, chardet.__version__) except (AssertionError, ValueError): - warnings.warn("urllib3 ({0}) or chardet ({1}) doesn't match a supported " - "version!".format(urllib3.__version__, chardet.__version__), - RequestsDependencyWarning) - + warnings.warn( + "urllib3 ({0}) or chardet ({1}) doesn't match a supported " + "version!".format(urllib3.__version__, chardet.__version__), + RequestsDependencyWarning, + ) # Attempt to enable urllib3's SNI support, if possible try: from urllib3.contrib import pyopenssl - pyopenssl.inject_into_urllib3() + pyopenssl.inject_into_urllib3() # Check cryptography version from cryptography import __version__ as cryptography_version + _check_cryptography(cryptography_version) except ImportError: pass - # urllib3's DependencyWarnings should be silenced. from urllib3.exceptions import DependencyWarning + warnings.simplefilter('ignore', DependencyWarning) from .__version__ import __title__, __description__, __url__, __version__ from .__version__ import __build__, __author__, __author_email__, __license__ from .__version__ import __copyright__, __cake__ -from . import utils +from .import utils from .models import Request, Response, PreparedRequest from .api import request, get, head, post, patch, put, delete, options from .sessions import session, Session from .status_codes import codes from .exceptions import ( - RequestException, Timeout, URLRequired, - TooManyRedirects, HTTPError, ConnectionError, - FileModeWarning, ConnectTimeout, ReadTimeout + RequestException, + Timeout, + URLRequired, + TooManyRedirects, + HTTPError, + ConnectionError, + FileModeWarning, + ConnectTimeout, + ReadTimeout, ) # Set default logging handler to avoid "No handler found" warnings. @@ -125,6 +131,5 @@ import logging from logging import NullHandler logging.getLogger(__name__).addHandler(NullHandler()) - # FileModeWarnings go off per the default. warnings.simplefilter('default', FileModeWarning, append=True) diff --git a/requests/__version__.py b/requests/__version__.py index 5347c7cc..ab45b99b 100644 --- a/requests/__version__.py +++ b/requests/__version__.py @@ -1,7 +1,6 @@ # .-. .-. .-. . . .-. .-. .-. .-. # |( |- |.| | | |- `-. | `-. # ' ' `-' `-`.`-' `-' `-' ' `-' - __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' diff --git a/requests/_internal_utils.py b/requests/_internal_utils.py index c611a2a3..51f32e69 100644 --- a/requests/_internal_utils.py +++ b/requests/_internal_utils.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests._internal_utils ~~~~~~~~~~~~~~ @@ -20,7 +19,6 @@ def to_native_string(string, encoding='ascii'): out = string else: out = string.decode(encoding) - return out @@ -35,5 +33,6 @@ def unicode_is_ascii(u_string): try: u_string.encode('ascii') return True + except UnicodeEncodeError: return False diff --git a/requests/adapters.py b/requests/adapters.py index c293287d..c0dd2d30 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.adapters ~~~~~~~~~~~~~~~~~ @@ -28,21 +27,35 @@ from urllib3.exceptions import ResponseError from .models import Response from .basics import urlparse, basestring -from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers, - prepend_scheme_if_needed, get_auth_from_url, urldefragauth, - select_proxy) +from .utils import ( + DEFAULT_CA_BUNDLE_PATH, + get_encoding_from_headers, + prepend_scheme_if_needed, + get_auth_from_url, + urldefragauth, + select_proxy, +) from .structures import CaseInsensitiveDict from .cookies import extract_cookies_to_jar -from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidScheme) +from .exceptions import ( + ConnectionError, + ConnectTimeout, + ReadTimeout, + SSLError, + ProxyError, + RetryError, + InvalidScheme, +) from .auth import _basic_auth_str try: from urllib3.contrib.socks import SOCKSProxyManager except ImportError: + def SOCKSProxyManager(*args, **kwargs): raise InvalidScheme("Missing dependencies for SOCKS support.") + DEFAULT_POOLBLOCK = False DEFAULT_POOLSIZE = 10 DEFAULT_RETRIES = 0 @@ -64,22 +77,19 @@ def _pool_kwargs(verify, cert): """ pool_kwargs = {} if verify: - cert_loc = None - # Allow self-specified cert location. if verify is not True: cert_loc = verify - if not cert_loc: cert_loc = DEFAULT_CA_BUNDLE_PATH - if not cert_loc or not os.path.exists(cert_loc): - raise IOError("Could not find a suitable TLS CA certificate bundle, " - "invalid path: {0}".format(cert_loc)) + raise IOError( + "Could not find a suitable TLS CA certificate bundle, " + "invalid path: {0}".format(cert_loc) + ) pool_kwargs['cert_reqs'] = 'CERT_REQUIRED' - if not os.path.isdir(cert_loc): pool_kwargs['ca_certs'] = cert_loc pool_kwargs['ca_cert_dir'] = None @@ -90,7 +100,6 @@ def _pool_kwargs(verify, cert): pool_kwargs['cert_reqs'] = 'CERT_NONE' pool_kwargs['ca_certs'] = None pool_kwargs['ca_cert_dir'] = None - if cert: if not isinstance(cert, basestring): pool_kwargs['cert_file'] = cert[0] @@ -98,15 +107,19 @@ def _pool_kwargs(verify, cert): else: pool_kwargs['cert_file'] = cert pool_kwargs['key_file'] = None - cert_file = pool_kwargs['cert_file'] key_file = pool_kwargs['key_file'] if cert_file and not os.path.exists(cert_file): - raise IOError("Could not find the TLS certificate file, " - "invalid path: {0}".format(cert_file)) + raise IOError( + "Could not find the TLS certificate file, " + "invalid path: {0}".format(cert_file) + ) + if key_file and not os.path.exists(key_file): - raise IOError("Could not find the TLS key file, " - "invalid path: {0}".format(key_file)) + raise IOError( + "Could not find the TLS key file, " "invalid path: {0}".format(key_file) + ) + return pool_kwargs @@ -116,8 +129,9 @@ class BaseAdapter(object): def __init__(self): super(BaseAdapter, self).__init__() - def send(self, request, stream=False, timeout=None, verify=True, - cert=None, proxies=None): + def send( + self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None + ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest ` being sent. @@ -165,25 +179,27 @@ class HTTPAdapter(BaseAdapter): >>> a = requests.adapters.HTTPAdapter(max_retries=3) >>> s.mount('http://', a) """ - __attrs__ = ['max_retries', 'config', '_pool_connections', '_pool_maxsize', - '_pool_block'] + __attrs__ = [ + 'max_retries', 'config', '_pool_connections', '_pool_maxsize', '_pool_block' + ] - def __init__(self, pool_connections=DEFAULT_POOLSIZE, - pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES, - pool_block=DEFAULT_POOLBLOCK): + def __init__( + self, + pool_connections=DEFAULT_POOLSIZE, + pool_maxsize=DEFAULT_POOLSIZE, + max_retries=DEFAULT_RETRIES, + pool_block=DEFAULT_POOLBLOCK, + ): if max_retries == DEFAULT_RETRIES: self.max_retries = Retry(0, read=False) else: self.max_retries = Retry.from_int(max_retries) self.config = {} self.proxy_manager = {} - super(HTTPAdapter, self).__init__() - self._pool_connections = pool_connections self._pool_maxsize = pool_maxsize self._pool_block = pool_block - self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) def __getstate__(self): @@ -194,14 +210,15 @@ class HTTPAdapter(BaseAdapter): # self.poolmanager uses a lambda function, which isn't pickleable. self.proxy_manager = {} self.config = {} - for attr, value in state.items(): setattr(self, attr, value) + self.init_poolmanager( + self._pool_connections, self._pool_maxsize, block=self._pool_block + ) - self.init_poolmanager(self._pool_connections, self._pool_maxsize, - block=self._pool_block) - - def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs): + def init_poolmanager( + self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs + ): """Initializes a urllib3 PoolManager. This method should not be called from user code, and is only @@ -217,9 +234,13 @@ class HTTPAdapter(BaseAdapter): self._pool_connections = connections self._pool_maxsize = maxsize self._pool_block = block - - self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize, - block=block, strict=True, **pool_kwargs) + self.poolmanager = PoolManager( + num_pools=connections, + maxsize=maxsize, + block=block, + strict=True, + **pool_kwargs, + ) def proxy_manager_for(self, proxy, **proxy_kwargs): """Return urllib3 ProxyManager for the given proxy. @@ -244,7 +265,7 @@ class HTTPAdapter(BaseAdapter): num_pools=self._pool_connections, maxsize=self._pool_maxsize, block=self._pool_block, - **proxy_kwargs + **proxy_kwargs, ) else: proxy_headers = self.proxy_headers(proxy) @@ -254,8 +275,8 @@ class HTTPAdapter(BaseAdapter): num_pools=self._pool_connections, maxsize=self._pool_maxsize, block=self._pool_block, - **proxy_kwargs) - + **proxy_kwargs, + ) return manager def build_response(self, req, resp): @@ -269,30 +290,23 @@ class HTTPAdapter(BaseAdapter): :rtype: requests.Response """ response = Response() - # Fallback to None if there's no status_code, for whatever reason. response.status_code = getattr(resp, 'status', None) - # Make headers case-insensitive. response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {})) - # Set encoding. response.encoding = get_encoding_from_headers(response.headers) response.raw = resp response.reason = response.raw.reason - if isinstance(req.url, bytes): response.url = req.url.decode('utf-8') else: response.url = req.url - # Add new cookies from the server. extract_cookies_to_jar(response.cookies, req, resp) - # Give the Response some context. response.request = req response.connection = self - return response def get_connection(self, url, proxies=None, verify=None, cert=None): @@ -306,7 +320,6 @@ class HTTPAdapter(BaseAdapter): """ pool_kwargs = _pool_kwargs(verify, cert) proxy = select_proxy(url, proxies) - if proxy: proxy = prepend_scheme_if_needed(proxy, 'http') proxy_manager = self.proxy_manager_for(proxy) @@ -316,7 +329,6 @@ class HTTPAdapter(BaseAdapter): parsed = urlparse(url) url = parsed.geturl() conn = self.poolmanager.connection_from_url(url, pool_kwargs=pool_kwargs) - return conn def close(self): @@ -345,17 +357,14 @@ class HTTPAdapter(BaseAdapter): """ proxy = select_proxy(request.url, proxies) scheme = urlparse(request.url).scheme - is_proxied_http_request = (proxy and scheme != 'https') using_socks_proxy = False if proxy: proxy_scheme = urlparse(proxy).scheme.lower() using_socks_proxy = proxy_scheme.startswith('socks') - url = request.path_url if is_proxied_http_request and not using_socks_proxy: url = urldefragauth(request.url) - return url def add_headers(self, request, **kwargs): @@ -387,14 +396,13 @@ class HTTPAdapter(BaseAdapter): """ headers = {} username, password = get_auth_from_url(proxy) - if username: - headers['Proxy-Authorization'] = _basic_auth_str(username, - password) - + headers['Proxy-Authorization'] = _basic_auth_str(username, password) return headers - def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None): + def send( + self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None + ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest ` being sent. @@ -411,27 +419,26 @@ class HTTPAdapter(BaseAdapter): :rtype: requests.Response """ conn = self.get_connection(request.url, proxies, verify, cert) - url = self.request_url(request, proxies) self.add_headers(request) - chunked = not (request.body is None or 'Content-Length' in request.headers) - if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError as e: # this may raise a string formatting error. - err = ("Invalid timeout {0}. Pass a (connect, read) " - "timeout tuple, or a single float to set " - "both timeouts to the same value".format(timeout)) + err = ( + "Invalid timeout {0}. Pass a (connect, read) " + "timeout tuple, or a single float to set " + "both timeouts to the same value".format(timeout) + ) raise ValueError(err) + elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) - try: if not chunked: resp = conn.urlopen( @@ -445,36 +452,28 @@ class HTTPAdapter(BaseAdapter): decode_content=False, retries=self.max_retries, timeout=timeout, - enforce_content_length=True + enforce_content_length=True, ) - # Send the request. else: if hasattr(conn, 'proxy_pool'): conn = conn.proxy_pool - low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT) - try: - low_conn.putrequest(request.method, - url, - skip_accept_encoding=True) - + low_conn.putrequest(request.method, url, skip_accept_encoding=True) for header, value in request.headers.items(): low_conn.putheader(header, value) - low_conn.endheaders() - for i in request.body: chunk_size = len(i) if chunk_size == 0: continue + low_conn.send(hex(chunk_size)[2:].encode('utf-8')) low_conn.send(b'\r\n') low_conn.send(i) low_conn.send(b'\r\n') low_conn.send(b'0\r\n\r\n') - # Receive the response from the server try: # For Python 2.7, use buffering of HTTP responses @@ -482,7 +481,6 @@ class HTTPAdapter(BaseAdapter): except TypeError: # For Python 3.3+ versions, this is the default r = low_conn.getresponse() - resp = HTTPResponse.from_httplib( r, pool=conn, @@ -490,7 +488,7 @@ class HTTPAdapter(BaseAdapter): preload_content=False, decode_content=False, enforce_content_length=True, - request_method=request.method + request_method=request.method, ) except: # If we hit any problems here, clean up the connection. @@ -529,8 +527,10 @@ class HTTPAdapter(BaseAdapter): if isinstance(e, _SSLError): # This branch is for urllib3 versions earlier than v1.22 raise SSLError(e, request=request) + elif isinstance(e, ReadTimeoutError): raise ReadTimeout(e, request=request) + else: raise diff --git a/requests/api.py b/requests/api.py index a121b0a3..d2fe9150 100644 --- a/requests/api.py +++ b/requests/api.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.api ~~~~~~~~~~~~ @@ -10,11 +9,13 @@ This module implements the Requests API. :license: Apache2, see LICENSE for more details. """ -from . import sessions -from . import types +from .import sessions +from .import types -def request(method: types.Method, url: types.URL, *, session: types.Session = None, **kwargs) -> types.Response: +def request( + method: types.Method, url: types.URL, *, session: types.Session = None, **kwargs +) -> types.Response: """Constructs and sends a :class:`Request `. :param method: method for the new :class:`Request` object. @@ -52,13 +53,10 @@ def request(method: types.Method, url: types.URL, *, session: types.Session = No >>> req = requests.request('GET', 'http://httpbin.org/get') """ - # By using the 'with' statement we are sure the session is closed, thus we # avoid leaving sockets open which can trigger a ResourceWarning in some # cases, and look like a memory leak in others. - session = sessions.Session() if session is None else session - with session: return session.request(method=method, url=url, **kwargs) @@ -72,7 +70,6 @@ def get(url: types.URL, *, params: types.Params = None, **kwargs) -> types.Respo :return: :class:`Response ` object :rtype: requests.Response """ - kwargs.setdefault('allow_redirects', True) return request('get', url, params=params, **kwargs) @@ -85,7 +82,6 @@ def options(url: types.URL, **kwargs) -> types.Response: :return: :class:`Response ` object :rtype: requests.Response """ - kwargs.setdefault('allow_redirects', True) return request('options', url, **kwargs) @@ -98,12 +94,13 @@ def head(url: types.URL, **kwargs) -> types.Response: :return: :class:`Response ` object :rtype: requests.Response """ - kwargs.setdefault('allow_redirects', False) return request('head', url, **kwargs) -def post(url: types.URL, *, data: types.Data = None, json: types.JSON = None, **kwargs) -> types.Response: +def post( + url: types.URL, *, data: types.Data = None, json: types.JSON = None, **kwargs +) -> types.Response: r"""Sends a POST request. :param url: URL for the new :class:`Request` object. @@ -113,7 +110,6 @@ def post(url: types.URL, *, data: types.Data = None, json: types.JSON = None, ** :return: :class:`Response ` object :rtype: requests.Response """ - return request('post', url, data=data, json=json, **kwargs) @@ -127,7 +123,6 @@ def put(url: types.URL, *, data: types.Data = None, **kwargs) -> types.Response: :return: :class:`Response ` object :rtype: requests.Response """ - return request('put', url, data=data, **kwargs) @@ -141,7 +136,6 @@ def patch(url: types.URL, *, data: types.Data = None, **kwargs) -> types.Respons :return: :class:`Response ` object :rtype: requests.Response """ - return request('patch', url, data=data, **kwargs) @@ -153,5 +147,4 @@ def delete(url: types.URL, **kwargs) -> types.Response: :return: :class:`Response ` object :rtype: requests.Response """ - return request('delete', url, **kwargs) diff --git a/requests/auth.py b/requests/auth.py index c15d3f2c..7bcf359e 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.auth ~~~~~~~~~~~~~ @@ -26,25 +25,25 @@ CONTENT_TYPE_MULTI_PART = 'multipart/form-data' def _basic_auth_str(username, password): """Returns a Basic Auth string.""" - if not isinstance(username, basestring): - raise TypeError('username must be of type str or bytes, ' - 'instead it was %s' % type(username)) + raise TypeError( + 'username must be of type str or bytes, ' + 'instead it was %s' % type(username) + ) if not isinstance(password, basestring): - raise TypeError('password must be of type str or bytes, ' - 'instead it was %s' % type(password)) + raise TypeError( + 'password must be of type str or bytes, ' + 'instead it was %s' % type(password) + ) if isinstance(username, str): username = username.encode('latin1') - if isinstance(password, str): password = password.encode('latin1') - authstr = 'Basic ' + to_native_string( b64encode(b':'.join((username, password))).strip() ) - return authstr @@ -63,10 +62,12 @@ class HTTPBasicAuth(AuthBase): self.password = password def __eq__(self, other): - return all([ - self.username == getattr(other, 'username', None), - self.password == getattr(other, 'password', None) - ]) + return all( + [ + self.username == getattr(other, 'username', None), + self.password == getattr(other, 'password', None), + ] + ) def __ne__(self, other): return not self == other @@ -99,51 +100,48 @@ class HTTPDigestAuth(AuthBase): """ :rtype: str """ - realm = self._thread_local.chal['realm'] nonce = self._thread_local.chal['nonce'] qop = self._thread_local.chal.get('qop') algorithm = self._thread_local.chal.get('algorithm') opaque = self._thread_local.chal.get('opaque') hash_utf8 = None - if algorithm is None: _algorithm = 'MD5' else: _algorithm = algorithm.upper() # lambdas assume digest modules are imported at the top level if _algorithm == 'MD5' or _algorithm == 'MD5-SESS': + def md5_utf8(x): if isinstance(x, str): x = x.encode('utf-8') return hashlib.md5(x).hexdigest() + hash_utf8 = md5_utf8 elif _algorithm == 'SHA': + def sha_utf8(x): if isinstance(x, str): x = x.encode('utf-8') return hashlib.sha1(x).hexdigest() + hash_utf8 = sha_utf8 - KD = lambda s, d: hash_utf8("%s:%s" % (s, d)) - if hash_utf8 is None: return None # XXX not implemented yet entdig = None p_parsed = urlparse(url) - #: path is request-uri defined in RFC 2616 which should not be empty + # : path is request-uri defined in RFC 2616 which should not be empty path = p_parsed.path or "/" if p_parsed.query: path += '?' + p_parsed.query - A1 = '%s:%s:%s' % (self.username, realm, self.password) A2 = '%s:%s' % (method, path) - HA1 = hash_utf8(A1) HA2 = hash_utf8(A2) - if nonce == self._thread_local.last_nonce: self._thread_local.nonce_count += 1 else: @@ -153,27 +151,23 @@ class HTTPDigestAuth(AuthBase): s += nonce.encode('utf-8') s += time.ctime().encode('utf-8') s += os.urandom(8) - cnonce = (hashlib.sha1(s).hexdigest()[:16]) if _algorithm == 'MD5-SESS': HA1 = hash_utf8('%s:%s:%s' % (HA1, nonce, cnonce)) - if not qop: respdig = KD(HA1, "%s:%s" % (nonce, HA2)) elif qop == 'auth' or 'auth' in qop.split(','): - noncebit = "%s:%s:%s:%s:%s" % ( - nonce, ncvalue, cnonce, 'auth', HA2 - ) + noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, 'auth', HA2) respdig = KD(HA1, noncebit) else: # XXX handle auth-int. return None self._thread_local.last_nonce = nonce - # XXX should the partial digests be encoded too? - base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \ - 'response="%s"' % (self.username, realm, nonce, path, respdig) + base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' 'response="%s"' % ( + self.username, realm, nonce, path, respdig + ) if opaque: base += ', opaque="%s"' % opaque if algorithm: @@ -182,7 +176,6 @@ class HTTPDigestAuth(AuthBase): base += ', digest="%s"' % entdig if qop: base += ', qop="auth", nc=%s, cnonce="%s"' % (ncvalue, cnonce) - return 'Digest %s' % (base) def handle_redirect(self, r, **kwargs): @@ -196,7 +189,6 @@ class HTTPDigestAuth(AuthBase): :rtype: requests.Response """ - # If response is not 4xx, do not auth # See https://github.com/requests/requests/issues/3772 if not 400 <= r.status_code < 500: @@ -208,13 +200,10 @@ class HTTPDigestAuth(AuthBase): # it was to resend the request. r.request.body.seek(self._thread_local.pos) s_auth = r.headers.get('www-authenticate', '') - if 'digest' in s_auth.lower() and self._thread_local.num_401_calls < 2: - self._thread_local.num_401_calls += 1 pat = re.compile(r'digest ', flags=re.IGNORECASE) self._thread_local.chal = parse_dict_header(pat.sub('', s_auth, count=1)) - # Consume content and release the original connection # to allow our new request to reuse the same one. r.content @@ -222,13 +211,12 @@ class HTTPDigestAuth(AuthBase): prep = r.request.copy() extract_cookies_to_jar(prep._cookies, r.request, r.raw) prep.prepare_cookies(prep._cookies) - prep.headers['Authorization'] = self.build_digest_header( - prep.method, prep.url) + prep.method, prep.url + ) _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep - return _r self._thread_local.num_401_calls = 1 @@ -251,14 +239,15 @@ class HTTPDigestAuth(AuthBase): r.register_hook('response', self.handle_401) r.register_hook('response', self.handle_redirect) self._thread_local.num_401_calls = 1 - return r def __eq__(self, other): - return all([ - self.username == getattr(other, 'username', None), - self.password == getattr(other, 'password', None) - ]) + return all( + [ + self.username == getattr(other, 'username', None), + self.password == getattr(other, 'password', None), + ] + ) def __ne__(self, other): return not self == other diff --git a/requests/basics.py b/requests/basics.py index bd59d50e..550c7022 100644 --- a/requests/basics.py +++ b/requests/basics.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.basics ~~~~~~~~~~~~~~~ @@ -14,23 +13,32 @@ import sys # --------- # Specifics # --------- - from urllib.parse import ( - urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, - quote_plus, unquote_plus, urldefrag + urlparse, + urlunparse, + urljoin, + urlsplit, + urlencode, + quote, + unquote, + quote_plus, + unquote_plus, + urldefrag, ) from urllib.request import ( - parse_http_list, getproxies, - proxy_bypass, proxy_bypass_environment, getproxies_environment + parse_http_list, + getproxies, + proxy_bypass, + proxy_bypass_environment, + getproxies_environment, ) from http import cookiejar as cookielib from http.cookies import Morsel from io import StringIO - -builtin_str = str # type: ignore -str = str # type: ignore -bytes = bytes # type: ignore +builtin_str = str # type: ignore +str = str # type: ignore +bytes = bytes # type: ignore basestring = (str, bytes) numeric_types = (int, float) integer_types = (int,) diff --git a/requests/certs.py b/requests/certs.py index d1a378d7..c811c194 100644 --- a/requests/certs.py +++ b/requests/certs.py @@ -1,6 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - """ requests.certs ~~~~~~~~~~~~~~ diff --git a/requests/cookies.py b/requests/cookies.py index 4291e4dc..adb61910 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.cookies ~~~~~~~~~~~~~~~~ @@ -20,7 +19,7 @@ from .basics import cookielib, urlparse, urlunparse, Morsel try: import threading except ImportError: - import dummy_threading as threading # type: ignore + import dummy_threading as threading # type: ignore class MockRequest(object): @@ -54,14 +53,21 @@ class MockRequest(object): # header if not self._r.headers.get('Host'): return self._r.url + # If they did set it, retrieve it and reconstruct the expected domain host = to_native_string(self._r.headers['Host'], encoding='utf-8') parsed = urlparse(self._r.url) # Reconstruct the URL as we expect it - return urlunparse([ - parsed.scheme, host, parsed.path, parsed.params, parsed.query, - parsed.fragment - ]) + return urlunparse( + [ + parsed.scheme, + host, + parsed.path, + parsed.params, + parsed.query, + parsed.fragment, + ] + ) def is_unverifiable(self): return True @@ -74,7 +80,9 @@ class MockRequest(object): def add_header(self, key, val): """cookielib has no legitimate use for this method; add it back if you find one.""" - raise NotImplementedError("Cookie headers should be added with add_unredirected_header()") + raise NotImplementedError( + "Cookie headers should be added with add_unredirected_header()" + ) def add_unredirected_header(self, name, value): self._new_headers[name] = value @@ -123,9 +131,9 @@ def extract_cookies_to_jar(jar, request, response): :param request: our own requests.Request object :param response: urllib3.HTTPResponse object """ - if not (hasattr(response, '_original_response') and - response._original_response): + if not (hasattr(response, '_original_response') and response._original_response): return + # the _original_response field is the wrapped httplib.HTTPResponse object, req = MockRequest(request) # pull out the HTTPMessage with the headers and put it in the mock: @@ -153,12 +161,14 @@ def remove_cookie_by_name(cookiejar, name, domain=None, path=None): for cookie in cookiejar: if cookie.name != name: continue + if domain is not None and domain != cookie.domain: continue + if path is not None and path != cookie.path: continue - clearables.append((cookie.domain, cookie.path, cookie.name)) + clearables.append((cookie.domain, cookie.path, cookie.name)) for domain, path, name in clearables: cookiejar.clear(domain, path, name) @@ -196,6 +206,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): """ try: return self._find_no_duplicates(name, domain, path) + except KeyError: return default @@ -206,7 +217,9 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): """ # support client code that unsets cookies by assignment of a None value: if value is None: - remove_cookie_by_name(self, name, domain=kwargs.get('domain'), path=kwargs.get('path')) + remove_cookie_by_name( + self, name, domain=kwargs.get('domain'), path=kwargs.get('path') + ) return if isinstance(value, Morsel): @@ -294,6 +307,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): for cookie in iter(self): if cookie.domain is not None and cookie.domain in domains: return True + domains.append(cookie.domain) return False # there is only one domain in jar @@ -316,6 +330,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): def __contains__(self, name): try: return super(RequestsCookieJar, self).__contains__(name) + except CookieConflictError: return True @@ -342,7 +357,11 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): remove_cookie_by_name(self, name) def set_cookie(self, cookie, *args, **kwargs): - if hasattr(cookie.value, 'startswith') and cookie.value.startswith('"') and cookie.value.endswith('"'): + if hasattr(cookie.value, 'startswith') and cookie.value.startswith( + '"' + ) and cookie.value.endswith( + '"' + ): cookie.value = cookie.value.replace('\\"', '') return super(RequestsCookieJar, self).set_cookie(cookie, *args, **kwargs) @@ -392,11 +411,14 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): if domain is None or cookie.domain == domain: if path is None or cookie.path == path: if toReturn is not None: # if there are multiple cookies that meet passed in criteria - raise CookieConflictError('There are multiple cookies with name, %r' % (name)) - toReturn = cookie.value # we will eventually return this as long as no cookie conflict + raise CookieConflictError( + 'There are multiple cookies with name, %r' % (name) + ) + toReturn = cookie.value # we will eventually return this as long as no cookie conflict if toReturn: return toReturn + raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path)) def __getstate__(self): @@ -426,6 +448,7 @@ def _copy_cookie_jar(jar): if hasattr(jar, 'copy'): # We're dealing with an instance of RequestsCookieJar return jar.copy() + # We're dealing with a generic CookieJar instance new_jar = copy.copy(jar) new_jar.clear() @@ -455,7 +478,6 @@ def create_cookie(name, value, **kwargs): 'rest': {'HttpOnly': None}, 'rfc2109': False, } - badargs = set(kwargs) - set(result) if badargs: err = 'create_cookie() got unexpected keyword arguments: %s' @@ -466,24 +488,21 @@ def create_cookie(name, value, **kwargs): result['domain_specified'] = bool(result['domain']) result['domain_initial_dot'] = result['domain'].startswith('.') result['path_specified'] = bool(result['path']) - return cookielib.Cookie(**result) def morsel_to_cookie(morsel): """Convert a Morsel object into a Cookie containing the one k/v pair.""" - expires = None if morsel['max-age']: try: expires = int(time.time() + int(morsel['max-age'])) except ValueError: raise TypeError('max-age: %s must be integer' % morsel['max-age']) + elif morsel['expires']: time_template = '%a, %d-%b-%Y %H:%M:%S GMT' - expires = calendar.timegm( - time.strptime(morsel['expires'], time_template) - ) + expires = calendar.timegm(time.strptime(morsel['expires'], time_template)) return create_cookie( comment=morsel['comment'], comment_url=bool(morsel['comment']), @@ -511,13 +530,11 @@ def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True): """ if cookiejar is None: cookiejar = RequestsCookieJar() - if cookie_dict is not None: names_from_jar = [cookie.name for cookie in cookiejar] for name in cookie_dict: if overwrite or (name not in names_from_jar): cookiejar.set_cookie(create_cookie(name, cookie_dict[name])) - return cookiejar @@ -531,13 +548,11 @@ def merge_cookies(cookiejar, cookies): raise ValueError('You can only merge into CookieJar') if isinstance(cookies, dict): - cookiejar = cookiejar_from_dict( - cookies, cookiejar=cookiejar, overwrite=False) + cookiejar = cookiejar_from_dict(cookies, cookiejar=cookiejar, overwrite=False) elif isinstance(cookies, cookielib.CookieJar): try: cookiejar.update(cookies) except AttributeError: for cookie_in_jar in cookies: cookiejar.set_cookie(cookie_in_jar) - return cookiejar diff --git a/requests/exceptions.py b/requests/exceptions.py index ebf4cc34..e4d3c366 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.exceptions ~~~~~~~~~~~~~~~~~~~ @@ -19,8 +18,7 @@ class RequestException(IOError): response = kwargs.pop('response', None) self.response = response self.request = kwargs.pop('request', None) - if (response is not None and not self.request and - hasattr(response, 'request')): + if (response is not None and not self.request and hasattr(response, 'request')): self.request = self.response.request super(RequestException, self).__init__(*args, **kwargs) @@ -108,9 +106,10 @@ class UnrewindableBodyError(RequestException): class InvalidBodyError(RequestException, ValueError): """An invalid request body was specified""" + + + # Warnings - - class RequestsWarning(Warning): """Base warning for Requests.""" pass diff --git a/requests/help.py b/requests/help.py index a590c946..6c299b34 100644 --- a/requests/help.py +++ b/requests/help.py @@ -10,12 +10,12 @@ import idna import urllib3 import chardet -from . import types +from .import types -from . import __version__ as requests_version +from .import __version__ as requests_version try: - from .packages.urllib3.contrib import pyopenssl + from . packages.urllib3.contrib import pyopenssl except ImportError: pyopenssl = None OpenSSL = None @@ -37,66 +37,47 @@ def _implementation() -> types.Help: to work out the correct shape of the code for those platforms. """ implementation = platform.python_implementation() - if implementation == 'CPython': implementation_version = platform.python_version() elif implementation == 'PyPy': - implementation_version = '%s.%s.%s' % (sys.pypy_version_info.major, - sys.pypy_version_info.minor, - sys.pypy_version_info.micro) + implementation_version = '%s.%s.%s' % ( + sys.pypy_version_info.major, + sys.pypy_version_info.minor, + sys.pypy_version_info.micro, + ) if sys.pypy_version_info.releaselevel != 'final': - implementation_version = ''.join([ - implementation_version, sys.pypy_version_info.releaselevel - ]) + implementation_version = ''.join( + [implementation_version, sys.pypy_version_info.releaselevel] + ) elif implementation == 'Jython': implementation_version = platform.python_version() # Complete Guess elif implementation == 'IronPython': implementation_version = platform.python_version() # Complete Guess else: implementation_version = 'Unknown' - return {'name': implementation, 'version': implementation_version} def info() -> types.Help: """Generate information for a bug report.""" try: - platform_info = { - 'system': platform.system(), - 'release': platform.release(), - } + platform_info = {'system': platform.system(), 'release': platform.release()} except IOError: - platform_info = { - 'system': 'Unknown', - 'release': 'Unknown', - } - + platform_info = {'system': 'Unknown', 'release': 'Unknown'} implementation_info = _implementation() urllib3_info = {'version': urllib3.__version__} chardet_info = {'version': chardet.__version__} - - pyopenssl_info = { - 'version': None, - 'openssl_version': '', - } + pyopenssl_info = {'version': None, 'openssl_version': ''} if OpenSSL: pyopenssl_info = { 'version': OpenSSL.__version__, 'openssl_version': '%x' % OpenSSL.SSL.OPENSSL_VERSION_NUMBER, } - cryptography_info = { - 'version': getattr(cryptography, '__version__', ''), - } - idna_info = { - 'version': getattr(idna, '__version__', ''), - } - + cryptography_info = {'version': getattr(cryptography, '__version__', '')} + idna_info = {'version': getattr(idna, '__version__', '')} # OPENSSL_VERSION_NUMBER doesn't exist in the Python 2.6 ssl module. system_ssl = getattr(ssl, 'OPENSSL_VERSION_NUMBER', None) - system_ssl_info = { - 'version': '%x' % system_ssl if system_ssl is not None else '' - } - + system_ssl_info = {'version': '%x' % system_ssl if system_ssl is not None else ''} return { 'platform': platform_info, 'implementation': implementation_info, @@ -107,9 +88,7 @@ def info() -> types.Help: 'chardet': chardet_info, 'cryptography': cryptography_info, 'idna': idna_info, - 'requests': { - 'version': requests_version, - }, + 'requests': {'version': requests_version}, } diff --git a/requests/hooks.py b/requests/hooks.py index 7a51f212..0a2a4dc7 100644 --- a/requests/hooks.py +++ b/requests/hooks.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.hooks ~~~~~~~~~~~~~~ @@ -17,9 +16,10 @@ HOOKS = ['response'] def default_hooks(): return {event: [] for event in HOOKS} + + + # TODO: response is the only one - - def dispatch_hook(key, hooks, hook_data, **kwargs): """Dispatches a hook dictionary on a given piece of data.""" hooks = hooks or {} diff --git a/requests/models.py b/requests/models.py index 5f6e922a..4bd0acef 100644 --- a/requests/models.py +++ b/requests/models.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.models ~~~~~~~~~~~~~~~ @@ -21,7 +20,8 @@ from urllib3.fields import RequestField from urllib3.filepost import encode_multipart_formdata from urllib3.util import parse_url from urllib3.exceptions import ( - DecodeError, ReadTimeoutError, ProtocolError, LocationParseError) + DecodeError, ReadTimeoutError, ProtocolError, LocationParseError +) from io import UnsupportedOperation from .hooks import default_hooks @@ -31,59 +31,74 @@ import requests from .auth import HTTPBasicAuth from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar from .exceptions import ( - HTTPError, MissingScheme, InvalidURL, ChunkedEncodingError, - ContentDecodingError, ConnectionError, StreamConsumedError, - InvalidHeader, InvalidBodyError, ReadTimeout + HTTPError, + MissingScheme, + InvalidURL, + ChunkedEncodingError, + ContentDecodingError, + ConnectionError, + StreamConsumedError, + InvalidHeader, + InvalidBodyError, + ReadTimeout, ) from ._internal_utils import to_native_string, unicode_is_ascii from .utils import ( - guess_filename, get_auth_from_url, requote_uri, - stream_decode_response_unicode, to_key_val_list, parse_header_links, - iter_slices, guess_json_utf, super_len, check_header_validity, - is_stream + guess_filename, + get_auth_from_url, + requote_uri, + stream_decode_response_unicode, + to_key_val_list, + parse_header_links, + iter_slices, + guess_json_utf, + super_len, + check_header_validity, + is_stream, ) from .basics import ( - cookielib, urlunparse, urlsplit, urlencode, str, bytes, - chardet, builtin_str, basestring + cookielib, + urlunparse, + urlsplit, + urlencode, + str, + bytes, + chardet, + builtin_str, + basestring, ) import json as complexjson from .status_codes import codes -#: The set of HTTP status codes that indicate an automatically +# : The set of HTTP status codes that indicate an automatically #: processable redirect. REDIRECT_STATI = ( - codes['moved'], # 301 - codes['found'], # 302 - codes['other'], # 303 + codes['moved'], # 301 + codes['found'], # 302 + codes['other'], # 303 codes['temporary_redirect'], # 307 codes['permanent_redirect'], # 308 ) - DEFAULT_REDIRECT_LIMIT = 30 CONTENT_CHUNK_SIZE = 10 * 1024 ITER_CHUNK_SIZE = 512 class RequestEncodingMixin(object): + @property def path_url(self): """Build the path URL to use.""" - url = [] - p = urlsplit(self.url) - path = p.path if not path: path = '/' - url.append(path) - query = p.query if query: url.append('?') url.append(query) - return ''.join(url) @staticmethod @@ -94,11 +109,12 @@ class RequestEncodingMixin(object): 2-tuples. Order is retained if data is a list of 2-tuples but arbitrary if parameters are supplied as a dict. """ - if isinstance(data, (str, bytes)): return data + elif hasattr(data, 'read'): return data + elif hasattr(data, '__iter__'): result = [] for k, vs in to_key_val_list(data): @@ -107,9 +123,13 @@ class RequestEncodingMixin(object): for v in vs: if v is not None: result.append( - (k.encode('utf-8') if isinstance(k, str) else k, - v.encode('utf-8') if isinstance(v, str) else v)) + ( + k.encode('utf-8') if isinstance(k, str) else k, + v.encode('utf-8') if isinstance(v, str) else v, + ) + ) return urlencode(result, doseq=True) + else: return data @@ -125,13 +145,13 @@ class RequestEncodingMixin(object): """ if (not files): raise ValueError("Files must be provided.") + elif isinstance(data, basestring): raise ValueError("Data must not be a string.") new_fields = [] fields = to_key_val_list(data or {}) files = to_key_val_list(files or {}) - for field, val in fields: if isinstance(val, basestring) or not hasattr(val, '__iter__'): val = [val] @@ -140,11 +160,14 @@ class RequestEncodingMixin(object): # Don't call str() on bytestrings: in Py3 it all goes wrong. if not isinstance(v, bytes): v = str(v) - new_fields.append( - (field.decode('utf-8') if isinstance(field, bytes) else field, - v.encode('utf-8') if isinstance(v, str) else v)) - + ( + field.decode('utf-8') if isinstance( + field, bytes + ) else field, + v.encode('utf-8') if isinstance(v, str) else v, + ) + ) for (k, v) in files: # support for explicit filename ft = None @@ -159,41 +182,41 @@ class RequestEncodingMixin(object): else: fn = guess_filename(v) or k fp = v - if isinstance(fp, (str, bytes, bytearray)): fdata = fp else: fdata = fp.read() - rf = RequestField(name=k, data=fdata, filename=fn, headers=fh) rf.make_multipart(content_type=ft) new_fields.append(rf) - body, content_type = encode_multipart_formdata(new_fields) - return body, content_type class RequestHooksMixin(object): + def register_hook(self, event, hook): """Properly register a hook.""" - if event not in self.hooks: - raise ValueError('Unsupported event specified, with event name "%s"' % (event)) + raise ValueError( + 'Unsupported event specified, with event name "%s"' % (event) + ) if isinstance(hook, collections.Callable): self.hooks[event].append(hook) elif hasattr(hook, '__iter__'): - self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable)) + self.hooks[event].extend( + h for h in hook if isinstance(h, collections.Callable) + ) def deregister_hook(self, event, hook): """Deregister a previously registered hook. Returns True if the hook existed, False if not. """ - try: self.hooks[event].remove(hook) return True + except ValueError: return False @@ -222,21 +245,28 @@ class Request(RequestHooksMixin): """ - def __init__(self, - method=None, url=None, headers=None, files=None, data=None, - params=None, auth=None, cookies=None, hooks=None, json=None): - + def __init__( + self, + method=None, + url=None, + headers=None, + files=None, + data=None, + params=None, + auth=None, + cookies=None, + hooks=None, + json=None, + ): # Default empty dicts for dict params. data = [] if data is None else data files = [] if files is None else files headers = {} if headers is None else headers params = {} if params is None else params hooks = {} if hooks is None else hooks - self.hooks = default_hooks() for (k, v) in list(hooks.items()): self.register_hook(event=k, hook=v) - self.method = method self.url = url self.headers = headers @@ -287,37 +317,44 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): """ def __init__(self): - #: HTTP verb to send to the server. + # : HTTP verb to send to the server. self.method = None - #: HTTP URL to send the request to. + # : HTTP URL to send the request to. self.url = None - #: dictionary of HTTP headers. + # : dictionary of HTTP headers. self.headers = None # The `CookieJar` used to create the Cookie header will be stored here # after prepare_cookies is called self._cookies = None - #: request body to send to the server. + # : request body to send to the server. self.body = None - #: dictionary of callback hooks, for internal usage. + # : dictionary of callback hooks, for internal usage. self.hooks = default_hooks() - #: integer denoting starting position of a readable file-like body. + # : integer denoting starting position of a readable file-like body. self._body_position = None - def prepare(self, - method=None, url=None, headers=None, files=None, data=None, - params=None, auth=None, cookies=None, hooks=None, json=None): + def prepare( + self, + method=None, + url=None, + headers=None, + files=None, + data=None, + params=None, + auth=None, + cookies=None, + hooks=None, + json=None, + ): """Prepares the entire request with the given parameters.""" - self.prepare_method(method) self.prepare_url(url, params) self.prepare_headers(headers) self.prepare_cookies(cookies) self.prepare_body(data, files, json) self.prepare_auth(auth, url) - # Note that prepare_auth must be last to enable authentication schemes # such as OAuth to work on a fully prepared request. - # This MUST go after prepare_auth. Authenticators could add a hook self.prepare_hooks(hooks) @@ -340,6 +377,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): self.method = method if self.method is None: raise ValueError('Request method cannot be "None"') + self.method = to_native_string(self.method.upper()) @staticmethod @@ -350,11 +388,12 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): host = idna.encode(host, uts46=True).decode('utf-8') except idna.IDNAError: raise UnicodeError + return host def prepare_url(self, url, params): """Prepares the given HTTP URL.""" - #: Accept objects that have string representations. + # : Accept objects that have string representations. #: We're unable to blindly call unicode/str functions #: as this will include the bytestring indicator (b'') #: on python 3.x. @@ -363,10 +402,8 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): url = url.decode('utf8') else: url = str(url) - # Ignore any leading and trailing whitespace characters. url = url.strip() - # Don't do any URL preparation for non-HTTP schemes like `mailto`, # `data` etc to work around exceptions from `url_parse`, which # handles RFC 3986 only. @@ -378,12 +415,13 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): try: scheme, auth, host, port, path, query, fragment = parse_url(url) except LocationParseError as e: - raise InvalidURL(*e.args) + raise InvalidURL(* e.args) if not scheme: - error = ("Invalid URL {0!r}: No scheme supplied. Perhaps you meant http://{0}?") + error = ( + "Invalid URL {0!r}: No scheme supplied. Perhaps you meant http://{0}?" + ) error = error.format(to_native_string(url, 'utf8')) - raise MissingScheme(error) if not host: @@ -398,6 +436,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): host = self._get_idna_encoded_host(host) except UnicodeError: raise InvalidURL('URL has an invalid label.') + elif host.startswith(u'*'): raise InvalidURL('URL has an invalid label.') @@ -408,27 +447,22 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): netloc += host if port: netloc += ':' + str(port) - # Bare domains aren't valid URLs. if not path: path = '/' - if isinstance(params, (str, bytes)): params = to_native_string(params) - enc_params = self._encode_params(params) if enc_params: if query: query = '%s&%s' % (query, enc_params) else: query = enc_params - url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment])) self.url = url def prepare_headers(self, headers): """Prepares the given HTTP headers.""" - self.headers = CaseInsensitiveDict() if headers: for header in headers.items(): @@ -439,14 +473,11 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): def prepare_body(self, data, files, json=None): """Prepares the given HTTP body data.""" - # Check if file, fo, generator, iterator. # If not, run through normal process. - # Nottin' on you. body = None content_type = None - if not data and json is not None: # urllib3 requires a bytes-like body. Python 2's json.dumps # provides this natively, but Python 3 gives a Unicode string. @@ -454,10 +485,8 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): body = complexjson.dumps(json) if not isinstance(body, bytes): body = body.encode('utf-8') - if is_stream(data): body = data - if getattr(body, 'tell', None) is not None: # Record the current file position before reading. # This will allow us to rewind a file in the event @@ -468,9 +497,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): # This differentiates from None, allowing us to catch # a failed `tell()` later when trying to rewind the body self._body_position = object() - if files: - raise NotImplementedError('Streamed bodies and files are mutually exclusive.') + raise NotImplementedError( + 'Streamed bodies and files are mutually exclusive.' + ) else: # Multi-part file uploads. @@ -483,11 +513,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): content_type = None else: content_type = 'application/x-www-form-urlencoded' - # Add content-type if it wasn't explicitly provided. if content_type and ('content-type' not in self.headers): self.headers['Content-Type'] = content_type - self.prepare_content_length(body) self.body = body @@ -502,41 +530,41 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): """ if body is not None: length = super_len(body) - if length: self.headers['Content-Length'] = builtin_str(length) elif is_stream(body): self.headers['Transfer-Encoding'] = 'chunked' else: - raise InvalidBodyError('Non-null body must have length or be streamable.') - elif self.method not in ('GET', 'HEAD') and self.headers.get('Content-Length') is None: + raise InvalidBodyError( + 'Non-null body must have length or be streamable.' + ) + + elif self.method not in ('GET', 'HEAD') and self.headers.get( + 'Content-Length' + ) is None: # Set Content-Length to 0 for methods that can have a body # but don't provide one. (i.e. not GET or HEAD) self.headers['Content-Length'] = '0' - if 'Transfer-Encoding' in self.headers and 'Content-Length' in self.headers: - raise InvalidHeader('Conflicting Headers: Both Transfer-Encoding and ' - 'Content-Length are set.') + raise InvalidHeader( + 'Conflicting Headers: Both Transfer-Encoding and ' + 'Content-Length are set.' + ) def prepare_auth(self, auth, url=''): """Prepares the given HTTP auth data.""" - # If no Auth is explicitly provided, extract it from the URL first. if auth is None: url_auth = get_auth_from_url(self.url) auth = url_auth if any(url_auth) else None - if auth: if isinstance(auth, tuple) and len(auth) == 2: # special-case basic HTTP auth auth = HTTPBasicAuth(*auth) - # Allow auth to make its changes. r = auth(self) - # Update self to reflect the auth changes. self.__dict__.update(r.__dict__) - # Recompute Content-Length self.prepare_content_length(self.body) @@ -555,7 +583,6 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): self._cookies = cookies else: self._cookies = cookiejar_from_dict(cookies) - cookie_header = get_cookie_header(self._cookies, self) if cookie_header is not None: self.headers['Cookie'] = cookie_header @@ -573,7 +600,6 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): """Sends the PreparedRequest to the given Session. If none is provided, one is created for you.""" session = requests.Session() if session is None else session - with session: return session.send(self, **send_kwargs) @@ -582,57 +608,54 @@ class Response(object): """The :class:`Response ` object, which contains a server's response to an HTTP request. """ - __attrs__ = [ - '_content', 'status_code', 'headers', 'url', 'history', - 'encoding', 'reason', 'cookies', 'elapsed', 'request' + '_content', + 'status_code', + 'headers', + 'url', + 'history', + 'encoding', + 'reason', + 'cookies', + 'elapsed', + 'request', ] def __init__(self): self._content = False self._content_consumed = False self._next = None - - #: Integer Code of responded HTTP Status, e.g. 404 or 200. + # : Integer Code of responded HTTP Status, e.g. 404 or 200. self.status_code = None - - #: Case-insensitive Dictionary of Response Headers. + # : Case-insensitive Dictionary of Response Headers. #: For example, ``headers['content-encoding']`` will return the #: value of a ``'Content-Encoding'`` response header. self.headers = CaseInsensitiveDict() - - #: File-like object representation of response (for advanced usage). + # : File-like object representation of response (for advanced usage). #: Use of ``raw`` requires that ``stream=True`` be set on the request. # This requirement does not apply for use internally to Requests. self.raw = None - - #: Final URL location of Response. + # : Final URL location of Response. self.url = None - - #: Encoding to decode with when accessing r.text or + # : Encoding to decode with when accessing r.text or #: r.iter_content(decode_unicode=True) self.encoding = None - - #: A list of :class:`Response ` objects from + # : A list of :class:`Response ` objects from #: the history of the Request. Any redirect responses will end #: up here. The list is sorted from the oldest to the most recent request. self.history = [] - - #: Textual reason of responded HTTP Status, e.g. "Not Found" or "OK". + # : Textual reason of responded HTTP Status, e.g. "Not Found" or "OK". self.reason = None - - #: A CookieJar of Cookies the server sent back. + # : A CookieJar of Cookies the server sent back. self.cookies = cookiejar_from_dict({}) - - #: The amount of time elapsed between sending the request + # : The amount of time elapsed between sending the request #: and the arrival of the response (as a timedelta). #: This property specifically measures the time taken between sending #: the first byte of the request and finishing parsing the headers. It #: is therefore unaffected by consuming the response content or the #: value of the ``stream`` keyword argument. self.elapsed = datetime.timedelta(0) - - #: The :class:`PreparedRequest ` object to which this + # : The :class:`PreparedRequest ` object to which this #: is a response. self.request = None @@ -647,13 +670,11 @@ class Response(object): # sure the content has been fully read. if not self._content_consumed: self.content - return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): for name, value in state.items(): setattr(self, name, value) - # pickled objects do not have .raw setattr(self, '_content_consumed', True) setattr(self, 'raw', None) @@ -678,6 +699,7 @@ class Response(object): self.raise_for_status() except HTTPError: return False + return True @property @@ -690,7 +712,10 @@ class Response(object): @property def is_permanent_redirect(self): """True if this Response one of the permanent versions of redirect.""" - return ('location' in self.headers and self.status_code in (codes.moved_permanently, codes.permanent_redirect)) + return ( + 'location' in self.headers and + self.status_code in (codes.moved_permanently, codes.permanent_redirect) + ) @property def next(self): @@ -725,52 +750,58 @@ class Response(object): try: for chunk in self.raw.stream(chunk_size, decode_content=True): yield chunk + except ProtocolError as e: if self.headers.get('Transfer-Encoding') == 'chunked': raise ChunkedEncodingError(e) + else: raise ConnectionError(e) + except DecodeError as e: raise ContentDecodingError(e) + except ReadTimeoutError as e: raise ReadTimeout(e) + else: # Standard file-like object. while True: chunk = self.raw.read(chunk_size) if not chunk: break + yield chunk self._content_consumed = True if self._content_consumed and isinstance(self._content, bool): raise StreamConsumedError() + elif chunk_size is not None and not isinstance(chunk_size, int): - raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size)) + raise TypeError( + "chunk_size must be an int, it is instead a %s." % type(chunk_size) + ) + # simulate reading small chunks of the content reused_chunks = iter_slices(self._content, chunk_size) - stream_chunks = generate() - chunks = reused_chunks if self._content_consumed else stream_chunks - if decode_unicode: if self.encoding is None: raise TypeError( - 'encoding must be set before consuming streaming ' - 'responses' + 'encoding must be set before consuming streaming ' 'responses' ) # check encoding value here, don't wait for the generator to be # consumed before raising an exception codecs.lookup(self.encoding) - chunks = stream_decode_response_unicode(chunks, self) - return chunks - def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None): + def iter_lines( + self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None + ): """Iterates over the response data, one line at a time. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. @@ -779,12 +810,11 @@ class Response(object): """ carriage_return = u'\r' if decode_unicode else b'\r' line_feed = u'\n' if decode_unicode else b'\n' - pending = None last_chunk_ends_with_cr = False - - for chunk in self.iter_content(chunk_size=chunk_size, - decode_unicode=decode_unicode): + for chunk in self.iter_content( + chunk_size=chunk_size, decode_unicode=decode_unicode + ): # Skip any null responses: if there is pending data it is necessarily an # incomplete chunk, so if we don't have more data we don't want to bother # trying to get it. Unconsumed pending data will be yielded anyway in the @@ -796,7 +826,6 @@ class Response(object): if pending is not None: chunk = pending + chunk pending = None - # Either split on a line, or split on a specified delimiter if delimiter: lines = chunk.split(delimiter) @@ -807,19 +836,20 @@ class Response(object): # starts with '\n', they should be merged and treated as only # *one* new line separator '\r\n' by splitlines(). # This rule only applies when splitlines() is used. - # The last chunk ends with '\r', so the '\n' at chunk[0] # is just the second half of a '\r\n' pair rather than a # new line break. Just skip it. - skip_first_char = last_chunk_ends_with_cr and chunk.startswith(line_feed) + skip_first_char = last_chunk_ends_with_cr and chunk.startswith( + line_feed + ) last_chunk_ends_with_cr = chunk.endswith(carriage_return) if skip_first_char: chunk = chunk[1:] # it's possible that after stripping the '\n' then chunk becomes empty if not chunk: continue - lines = chunk.splitlines() + lines = chunk.splitlines() # Calling `.split(delimiter)` will always end with whatever text # remains beyond the delimiter, or '' if the delimiter is the end # of the text. On the other hand, `.splitlines()` doesn't include @@ -838,7 +868,6 @@ class Response(object): incomplete_line = lines[-1] and lines[-1][-1] == chunk[-1] if delimiter or incomplete_line: pending = lines.pop() - for line in lines: yield line @@ -848,18 +877,18 @@ class Response(object): @property def content(self): """Content of the response, in bytes.""" - if self._content is False: # Read the contents. if self._content_consumed: - raise RuntimeError( - 'The content for this response was already consumed') + raise RuntimeError('The content for this response was already consumed') if self.status_code == 0 or self.raw is None: self._content = None else: - self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes() - + self._content = bytes().join( + self.iter_content(CONTENT_CHUNK_SIZE) + ) or bytes( + ) self._content_consumed = True # don't need to release the connection; that's been handled by urllib3 # since we exhausted the data. @@ -877,18 +906,15 @@ class Response(object): non-HTTP knowledge to make a better guess at the encoding, you should set ``r.encoding`` appropriately before accessing this property. """ - # Try charset from content-type content = None encoding = self.encoding - if not self.content: return str('') # Fallback to auto-detected encoding. if self.encoding is None: encoding = self.apparent_encoding - # Decode unicode from given encoding. try: content = str(self.content, encoding, errors='replace') @@ -900,7 +926,6 @@ class Response(object): # # So we try blindly encoding. content = str(self.content, errors='replace') - return content def json(self, **kwargs): @@ -909,7 +934,6 @@ class Response(object): :param \*\*kwargs: Optional arguments that ``json.loads`` takes. :raises ValueError: If the response body does not contain valid json. """ - if not self.encoding and self.content and len(self.content) > 3: # No encoding set. JSON RFC 4627 section 3 states we should expect # UTF-8, -16 or -32. Detect which one to use; If the detection or @@ -918,9 +942,8 @@ class Response(object): encoding = guess_json_utf(self.content) if encoding is not None: try: - return complexjson.loads( - self.content.decode(encoding), **kwargs - ) + return complexjson.loads(self.content.decode(encoding), **kwargs) + except UnicodeDecodeError: # Wrong UTF codec detected; usually because it's not UTF-8 # but some other 8-bit codec. This is an RFC violation, @@ -932,25 +955,19 @@ class Response(object): @property def links(self): """Returns the parsed header links of the response, if any.""" - header = self.headers.get('link') - # l = MultiDict() l = {} - if header: links = parse_header_links(header) - for link in links: key = link.get('rel') or link.get('url') l[key] = link - return l def raise_for_status(self): """Raises stored :class:`HTTPError`, if one occurred. Otherwise, returns the response object (self).""" - http_error_msg = '' if isinstance(self.reason, bytes): # We attempt to decode utf-8 first because some servers @@ -963,13 +980,14 @@ class Response(object): reason = self.reason.decode('iso-8859-1') else: reason = self.reason - if 400 <= self.status_code < 500: - http_error_msg = u'%s Client Error: %s for url: %s' % (self.status_code, reason, self.url) - + http_error_msg = u'%s Client Error: %s for url: %s' % ( + self.status_code, reason, self.url + ) elif 500 <= self.status_code < 600: - http_error_msg = u'%s Server Error: %s for url: %s' % (self.status_code, reason, self.url) - + http_error_msg = u'%s Server Error: %s for url: %s' % ( + self.status_code, reason, self.url + ) if http_error_msg: raise HTTPError(http_error_msg, response=self) @@ -983,7 +1001,6 @@ class Response(object): """ if not self._content_consumed: self.raw.close() - release_conn = getattr(self.raw, 'release_conn', None) if release_conn is not None: release_conn() diff --git a/requests/sessions.py b/requests/sessions.py index 7eb7d2ec..477dbb11 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.session ~~~~~~~~~~~~~~~~ @@ -16,22 +15,36 @@ from datetime import timedelta from .auth import _basic_auth_str from .basics import cookielib, urljoin, urlparse, str from .cookies import ( - cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, - merge_cookies, _copy_cookie_jar) + cookiejar_from_dict, + extract_cookies_to_jar, + RequestsCookieJar, + merge_cookies, + _copy_cookie_jar, +) from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT from .hooks import default_hooks, dispatch_hook from ._internal_utils import to_native_string from .utils import to_key_val_list, default_headers from .exceptions import ( - TooManyRedirects, InvalidScheme, ChunkedEncodingError, - ConnectionError, ContentDecodingError, InvalidHeader) + TooManyRedirects, + InvalidScheme, + ChunkedEncodingError, + ConnectionError, + ContentDecodingError, + InvalidHeader, +) from .structures import CaseInsensitiveDict from .adapters import HTTPAdapter from .utils import ( - requote_uri, get_environ_proxies, get_netrc_auth, should_bypass_proxies, - get_auth_from_url, is_valid_location, rewind_body + requote_uri, + get_environ_proxies, + get_netrc_auth, + should_bypass_proxies, + get_auth_from_url, + is_valid_location, + rewind_body, ) from .status_codes import codes @@ -54,7 +67,6 @@ def merge_setting(request_setting, session_setting, dict_class=OrderedDict): the explicit setting on that request, and the setting in the session. If a setting is a dictionary, they will be merged together using `dict_class`. """ - if session_setting is None: return request_setting @@ -63,20 +75,17 @@ def merge_setting(request_setting, session_setting, dict_class=OrderedDict): # Bypass if not a dictionary (e.g. verify) if not ( - isinstance(session_setting, Mapping) and - isinstance(request_setting, Mapping) + isinstance(session_setting, Mapping) and isinstance(request_setting, Mapping) ): return request_setting merged_setting = dict_class(to_key_val_list(session_setting)) merged_setting.update(to_key_val_list(request_setting)) - # Remove keys that are set to None. Extract keys first to avoid altering # the dictionary during iteration. none_keys = [k for (k, v) in merged_setting.items() if v is None] for key in none_keys: del merged_setting[key] - return merged_setting @@ -107,11 +116,12 @@ class SessionRedirectMixin(object): # attribute. if response.is_redirect: if not is_valid_location(response): - raise InvalidHeader('Response contains multiple Location headers. ' - 'Unable to perform redirect.') + raise InvalidHeader( + 'Response contains multiple Location headers. ' + 'Unable to perform redirect.' + ) location = response.headers['location'] - # Currently the underlying http module on py3 decode headers # in latin1, but empirical evidence suggests that latin1 is very # rarely used with non-ASCII characters in HTTP headers. @@ -120,43 +130,56 @@ class SessionRedirectMixin(object): # To solve this, we re-encode the location in latin1. location = location.encode('latin1') return to_native_string(location, 'utf8') + return None - def resolve_redirects(self, response, request, stream=False, timeout=None, - verify=True, cert=None, proxies=None, - yield_requests=False, **adapter_kwargs): + def resolve_redirects( + self, + response, + request, + stream=False, + timeout=None, + verify=True, + cert=None, + proxies=None, + yield_requests=False, + **adapter_kwargs, + ): """Given a Response, yields Responses until 'Location' header-based redirection ceases, or the Session.max_redirects limit has been reached. """ - - history = [response] # keep track of history; seed it with the original response - + history = [ + response + ] # keep track of history; seed it with the original response location_url = self.get_redirect_target(response) - while location_url: prepared_request = request.copy() - try: response.content # Consume socket so it can be released - except (ChunkedEncodingError, ConnectionError, ContentDecodingError, RuntimeError): + except ( + ChunkedEncodingError, + ConnectionError, + ContentDecodingError, + RuntimeError, + ): response.raw.read(decode_content=False) - if len(response.history) >= self.max_redirects: - raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects, response=response) + raise TooManyRedirects( + 'Exceeded %s redirects.' % self.max_redirects, response=response + ) # Release the connection back into the pool. response.close() - # Handle redirection without scheme (see: RFC 1808 Section 4) if location_url.startswith('//'): parsed_rurl = urlparse(response.url) - location_url = '%s:%s' % (to_native_string(parsed_rurl.scheme), location_url) - + location_url = '%s:%s' % ( + to_native_string(parsed_rurl.scheme), location_url + ) # The scheme should be lower case... parsed = urlparse(location_url) location_url = parsed.geturl() - # Facilitate relative 'location' headers, as allowed by RFC 7231. # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') # Compliant with RFC3986, we percent encode the url. @@ -164,11 +187,8 @@ class SessionRedirectMixin(object): location_url = urljoin(response.url, requote_uri(location_url)) else: location_url = requote_uri(location_url) - prepared_request.url = to_native_string(location_url) - method_changed = self.rebuild_method(prepared_request, response) - # https://github.com/kennethreitz/requests/issues/2590 # If method is changed to GET we need to remove body and associated headers. if method_changed and prepared_request.method == 'GET': @@ -177,24 +197,20 @@ class SessionRedirectMixin(object): for header in purged_headers: prepared_request.headers.pop(header, None) prepared_request.body = None - headers = prepared_request.headers try: del headers['Cookie'] except KeyError: pass - # Extract any cookies sent on the response to the cookiejar # in the new request. Because we've mutated our copied prepared # request, use the old one that we haven't yet touched. extract_cookies_to_jar(prepared_request._cookies, request, response.raw) merge_cookies(prepared_request._cookies, self.cookies) prepared_request.prepare_cookies(prepared_request._cookies) - # Rebuild auth and proxy information. proxies = self.rebuild_proxies(prepared_request, proxies) self.rebuild_auth(prepared_request, response) - # A failed tell() sets `_body_position` to `object()`. This non-None # value ensures `rewindable` will be True, allowing us to raise an # UnrewindableBodyError, instead of hanging the connection. @@ -202,18 +218,15 @@ class SessionRedirectMixin(object): prepared_request._body_position is not None and ('Content-Length' in headers or 'Transfer-Encoding' in headers) ) - # Attempt to rewind consumed file-like object. if rewindable: rewind_body(prepared_request) - # Override the original request. request = prepared_request - if yield_requests: yield request - else: + else: response = self.send( request, stream=stream, @@ -222,15 +235,13 @@ class SessionRedirectMixin(object): cert=cert, proxies=proxies, allow_redirects=False, - **adapter_kwargs + **adapter_kwargs, ) # copy our history tracker into the response response.history = history[:] # append the new response to the history tracker for the next iteration history.append(response) - extract_cookies_to_jar(self.cookies, prepared_request, response.raw) - # extract redirect url, if any, for the next loop location_url = self.get_redirect_target(response) yield response @@ -243,21 +254,17 @@ class SessionRedirectMixin(object): """ headers = prepared_request.headers url = prepared_request.url - if 'Authorization' in headers: # If we get redirected to a new host, we should strip out any # authentication headers. original_parsed = urlparse(response.request.url) redirect_parsed = urlparse(url) - if (original_parsed.hostname != redirect_parsed.hostname): del headers['Authorization'] - # .netrc might have more auth for us on our new host. new_auth = get_netrc_auth(url) if self.trust_env else None if new_auth is not None: prepared_request.prepare_auth(new_auth) - return def rebuild_proxies(self, prepared_request, proxies): @@ -278,27 +285,20 @@ class SessionRedirectMixin(object): scheme = urlparse(url).scheme new_proxies = proxies.copy() no_proxy = proxies.get('no_proxy') - bypass_proxy = should_bypass_proxies(url, no_proxy=no_proxy) if self.trust_env and not bypass_proxy: environ_proxies = get_environ_proxies(url, no_proxy=no_proxy) - proxy = environ_proxies.get(scheme, environ_proxies.get('all')) - if proxy: new_proxies.setdefault(scheme, proxy) - if 'Proxy-Authorization' in headers: del headers['Proxy-Authorization'] - try: username, password = get_auth_from_url(new_proxies[scheme]) except KeyError: username, password = None, None - if username and password: headers['Proxy-Authorization'] = _basic_auth_str(username, password) - return new_proxies def rebuild_method(self, prepared_request, response): @@ -309,11 +309,9 @@ class SessionRedirectMixin(object): :return: boolean expressing if the method changed during rebuild. """ method = original_method = prepared_request.method - # http://tools.ietf.org/html/rfc7231#section-6.4.4 if response.status_code == codes.see_other and method != 'HEAD': method = 'GET' - # If a POST is responded to with a 301 or 302, turn it into a GET. This has # become a common pattern in browsers and was introduced into later versions # of HTTP RFCs. While some browsers transform other methods to GET, little of @@ -321,7 +319,6 @@ class SessionRedirectMixin(object): # which only supports POST->GET. if response.status_code in (codes.found, codes.moved) and method == 'POST': method = 'GET' - prepared_request.method = method return method != original_method @@ -344,63 +341,60 @@ class Session(SessionRedirectMixin): >>> s.get('http://httpbin.org/get') """ - __attrs__ = [ - 'headers', 'cookies', 'auth', 'proxies', 'hooks', 'params', 'verify', - 'cert', 'prefetch', 'adapters', 'stream', 'trust_env', + 'headers', + 'cookies', + 'auth', + 'proxies', + 'hooks', + 'params', + 'verify', + 'cert', + 'prefetch', + 'adapters', + 'stream', + 'trust_env', 'max_redirects', ] def __init__(self): - - #: A case-insensitive dictionary of headers to be sent on each + # : A case-insensitive dictionary of headers to be sent on each #: :class:`Request ` sent from this #: :class:`Session `. self.headers = default_headers() - - #: Default Authentication tuple or object to attach to + # : Default Authentication tuple or object to attach to #: :class:`Request `. self.auth = None - - #: Dictionary mapping protocol or protocol and host to the URL of the proxy + # : Dictionary mapping protocol or protocol and host to the URL of the proxy #: (e.g. {'http': 'foo.bar:3128', 'http://host.name': 'foo.bar:4012'}) to #: be used on each :class:`Request `. self.proxies = {} - - #: Event-handling hooks. + # : Event-handling hooks. self.hooks = default_hooks() - - #: Dictionary of querystring data to attach to each + # : Dictionary of querystring data to attach to each #: :class:`Request `. The dictionary values may be lists for #: representing multivalued query parameters. self.params = {} - - #: Stream response content default. + # : Stream response content default. self.stream = False - - #: SSL Verification default. + # : SSL Verification default. self.verify = True - - #: SSL client certificate default, if String, path to ssl client + # : SSL client certificate default, if String, path to ssl client #: cert file (.pem). If Tuple, ('cert', 'key') pair. self.cert = None - - #: Maximum number of redirects allowed. If the request exceeds this + # : Maximum number of redirects allowed. If the request exceeds this #: limit, a :class:`TooManyRedirects` exception is raised. #: This defaults to requests.models.DEFAULT_REDIRECT_LIMIT, which is #: 30. self.max_redirects = DEFAULT_REDIRECT_LIMIT - - #: Trust environment settings for proxy configuration, default + # : Trust environment settings for proxy configuration, default #: authentication and similar. self.trust_env = True - - #: A CookieJar containing all currently outstanding cookies set on this + # : A CookieJar containing all currently outstanding cookies set on this #: session. By default it is a #: :class:`RequestsCookieJar `, but #: may be any other ``cookielib.CookieJar`` compatible object. self.cookies = cookiejar_from_dict({}) - # Default connection adapters. self.adapters = OrderedDict() self.mount('https://', HTTPAdapter()) @@ -423,20 +417,16 @@ class Session(SessionRedirectMixin): :rtype: requests.PreparedRequest """ cookies = request.cookies or {} - # Bootstrap CookieJar. if not isinstance(cookies, cookielib.CookieJar): cookies = cookiejar_from_dict(cookies) - # Merge with session cookies session_cookies = _copy_cookie_jar(self.cookies) merged_cookies = merge_cookies(session_cookies, cookies) - # Set environment's basic authentication if not explicitly set. auth = request.auth if self.trust_env and not auth and not self.auth: auth = get_netrc_auth(request.url) - p = PreparedRequest() p.prepare( method=request.method.upper(), @@ -444,7 +434,9 @@ class Session(SessionRedirectMixin): files=request.files, data=request.data, json=request.json, - headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict), + headers=merge_setting( + request.headers, self.headers, dict_class=CaseInsensitiveDict + ), params=merge_setting(request.params, self.params), auth=merge_setting(auth, self.auth), cookies=merged_cookies, @@ -452,10 +444,25 @@ class Session(SessionRedirectMixin): ) return p - def request(self, method, url, - params=None, data=None, headers=None, cookies=None, files=None, - auth=None, timeout=None, allow_redirects=True, proxies=None, - hooks=None, stream=None, verify=None, cert=None, json=None): + def request( + self, + method, + url, + params=None, + data=None, + headers=None, + cookies=None, + files=None, + auth=None, + timeout=None, + allow_redirects=True, + proxies=None, + hooks=None, + stream=None, + verify=None, + cert=None, + json=None, + ): """Constructs a :class:`Request `, prepares it, and sends it. Returns :class:`Response ` object. @@ -506,21 +513,14 @@ class Session(SessionRedirectMixin): hooks=hooks, ) prep = self.prepare_request(req) - proxies = proxies or {} - settings = self.merge_environment_settings( prep.url, proxies, stream, verify, cert ) - # Send the request. - send_kwargs = { - 'timeout': timeout, - 'allow_redirects': allow_redirects, - } + send_kwargs = {'timeout': timeout, 'allow_redirects': allow_redirects} send_kwargs.update(settings) resp = self.send(prep, **send_kwargs) - return resp def get(self, url, **kwargs): @@ -530,7 +530,6 @@ class Session(SessionRedirectMixin): :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ - kwargs.setdefault('allow_redirects', True) return self.request('GET', url, **kwargs) @@ -541,7 +540,6 @@ class Session(SessionRedirectMixin): :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ - kwargs.setdefault('allow_redirects', True) return self.request('OPTIONS', url, **kwargs) @@ -552,7 +550,6 @@ class Session(SessionRedirectMixin): :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ - kwargs.setdefault('allow_redirects', False) return self.request('HEAD', url, **kwargs) @@ -565,7 +562,6 @@ class Session(SessionRedirectMixin): :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ - return self.request('POST', url, data=data, json=json, **kwargs) def put(self, url, data=None, **kwargs): @@ -576,7 +572,6 @@ class Session(SessionRedirectMixin): :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ - return self.request('PUT', url, data=data, **kwargs) def patch(self, url, data=None, **kwargs): @@ -587,7 +582,6 @@ class Session(SessionRedirectMixin): :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ - return self.request('PATCH', url, data=data, **kwargs) def delete(self, url, **kwargs): @@ -597,7 +591,6 @@ class Session(SessionRedirectMixin): :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ - return self.request('DELETE', url, **kwargs) def send(self, request, **kwargs): @@ -611,7 +604,6 @@ class Session(SessionRedirectMixin): kwargs.setdefault('verify', self.verify) kwargs.setdefault('cert', self.cert) kwargs.setdefault('proxies', self.proxies) - # It's possible that users might accidentally send a Request object. # Guard against that specific failure case. if isinstance(request, Request): @@ -622,52 +614,40 @@ class Session(SessionRedirectMixin): allow_redirects = kwargs.pop('allow_redirects', True) stream = kwargs.get('stream') hooks = request.hooks - # Get the appropriate adapter to use adapter = self.get_adapter(url=request.url) - # Start time (approximately) of the request start = preferred_clock() - # Send the request r = adapter.send(request, **kwargs) - # Total elapsed time of the request (approximately) elapsed = preferred_clock() - start r.elapsed = timedelta(seconds=elapsed) - # Response manipulation hooks. r = dispatch_hook('response', hooks, r, **kwargs) - # Persist cookies if r.history: - # If the hooks create history then we want those cookies too for resp in r.history: extract_cookies_to_jar(self.cookies, resp.request, resp.raw) - extract_cookies_to_jar(self.cookies, request, r.raw) - # Redirect resolving generator. gen = self.resolve_redirects(r, request, **kwargs) - # Resolve redirects, if allowed. history = [resp for resp in gen] if allow_redirects else [] - # If there is a history, replace ``r`` with the last response if history: r = history.pop() - # If redirects aren't being followed, store the response on the Request for Response.next(). if not allow_redirects: try: - r._next = next(self.resolve_redirects(r, request, yield_requests=True, **kwargs)) + r._next = next( + self.resolve_redirects(r, request, yield_requests=True, **kwargs) + ) except StopIteration: pass - if not stream: r.content - return r def merge_environment_settings(self, url, proxies, stream, verify, cert): @@ -687,10 +667,11 @@ class Session(SessionRedirectMixin): # Look for requests environment configuration and be compatible # with cURL. if verify is True or verify is None: - verify = (os.environ.get('REQUESTS_CA_BUNDLE') or - os.environ.get('CURL_CA_BUNDLE') or - verify) - + verify = ( + os.environ.get('REQUESTS_CA_BUNDLE') or + os.environ.get('CURL_CA_BUNDLE') or + verify + ) # Now we handle proxies. # Proxies need to be built up backwards. This is because None values # can delete proxy information, which can then be re-added by a more @@ -699,17 +680,12 @@ class Session(SessionRedirectMixin): no_proxy = proxies.get('no_proxy') if proxies is not None else None if no_proxy is None: no_proxy = self.proxies.get('no_proxy') - env_proxies = {} - if self.trust_env: env_proxies = get_environ_proxies(url, no_proxy=no_proxy) or {} - new_proxies = merge_setting(self.proxies, env_proxies) proxies = merge_setting(proxies, new_proxies) - - return {'verify': verify, 'proxies': proxies, 'stream': stream, - 'cert': cert} + return {'verify': verify, 'proxies': proxies, 'stream': stream, 'cert': cert} def get_adapter(self, url): """ @@ -718,7 +694,6 @@ class Session(SessionRedirectMixin): :rtype: requests.adapters.BaseAdapter """ for (prefix, adapter) in self.adapters.items(): - if url.lower().startswith(prefix): return adapter @@ -737,7 +712,6 @@ class Session(SessionRedirectMixin): """ self.adapters[prefix] = adapter keys_to_move = [k for k in self.adapters if len(k) < len(prefix)] - for key in keys_to_move: self.adapters[key] = self.adapters.pop(key) @@ -756,5 +730,4 @@ def session(): :rtype: Session """ - return Session() diff --git a/requests/status_codes.py b/requests/status_codes.py index 289e1a24..964cb8c3 100644 --- a/requests/status_codes.py +++ b/requests/status_codes.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- - from .structures import LookupDict _codes = { - # Informational. 100: ('continue',), 101: ('switching_protocols',), @@ -20,7 +18,6 @@ _codes = { 207: ('multi_status', 'multiple_status', 'multi_stati', 'multiple_stati'), 208: ('already_reported',), 226: ('im_used',), - # Redirection. 300: ('multiple_choices',), 301: ('moved_permanently', 'moved', '\\o-'), @@ -30,9 +27,8 @@ _codes = { 305: ('use_proxy',), 306: ('switch_proxy',), 307: ('temporary_redirect', 'temporary_moved', 'temporary'), - 308: ('permanent_redirect', - 'resume_incomplete', 'resume',), # These 2 to be removed in 3.0 - + 308: ('permanent_redirect', 'resume_incomplete', 'resume'), + # These 2 to be removed in 3.0 # Client Error. 400: ('bad_request', 'bad'), 401: ('unauthorized',), @@ -50,7 +46,9 @@ _codes = { 413: ('request_entity_too_large',), 414: ('request_uri_too_large',), 415: ('unsupported_media_type', 'unsupported_media', 'media_type'), - 416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'), + 416: ( + 'requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable' + ), 417: ('expectation_failed',), 418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'), 421: ('misdirected_request',), @@ -67,7 +65,6 @@ _codes = { 450: ('blocked_by_windows_parental_controls', 'parental_controls'), 451: ('unavailable_for_legal_reasons', 'legal_reasons'), 499: ('client_closed_request',), - # Server Error. 500: ('internal_server_error', 'server_error', '/o\\', '✗'), 501: ('not_implemented',), @@ -81,11 +78,9 @@ _codes = { 510: ('not_extended',), 511: ('network_authentication_required', 'network_auth', 'network_authentication'), } - codes = LookupDict(name='status_codes') - for code, titles in _codes.items(): - for title in titles: # type: ignore + for title in titles: # type: ignore setattr(codes, title, code) if not title.startswith(('\\', '/')): setattr(codes, title.upper(), code) diff --git a/requests/structures.py b/requests/structures.py index 94f01c80..c0b13c2b 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.structures ~~~~~~~~~~~~~~~~~~~ @@ -62,20 +61,18 @@ class CaseInsensitiveDict(collections.MutableMapping): def lower_items(self): """Like iteritems(), but with all lowercase keys.""" - return ( - (lowerkey, keyval[1]) - for (lowerkey, keyval) - in self._store.items() - ) + return ((lowerkey, keyval[1]) for (lowerkey, keyval) in self._store.items()) def __eq__(self, other): if isinstance(other, collections.Mapping): other = CaseInsensitiveDict(other) else: return NotImplemented + # Compare insensitively return dict(self.lower_items()) == dict(other.lower_items()) + # Copy is required def copy(self): return CaseInsensitiveDict(self._store.values()) @@ -96,7 +93,6 @@ class LookupDict(dict): def __getitem__(self, key): # We allow fall-through here, so values default to None - return self.__dict__.get(key, None) def __iter__(self): diff --git a/requests/types.py b/requests/types.py index 1744933a..d432fb0d 100644 --- a/requests/types.py +++ b/requests/types.py @@ -1,24 +1,36 @@ from typing import ( - Callable, Optional, Union, Any, Iterable, List, Mapping, MutableMapping, - Tuple, IO, Text, Type, Dict + Callable, + Optional, + Union, + Any, + Iterable, + List, + Mapping, + MutableMapping, + Tuple, + IO, + Text, + Type, + Dict, ) -from . import auth +from .import auth from .models import Response, PreparedRequest from .cookies import RequestsCookieJar from .sessions import Session -_ParamsMappingValueType = Union[str, bytes, int, float, Iterable[Union[str, bytes, int, float]]] +_ParamsMappingValueType = Union[ + str, bytes, int, float, Iterable[Union[str, bytes, int, float]] +] Params = Optional[ Union[ - Mapping[ - Union[str, bytes, int, float], _ParamsMappingValueType], - Union[str, bytes], - Tuple[Union[str, bytes, int, float], _ParamsMappingValueType], - Mapping[str, _ParamsMappingValueType], - Mapping[bytes, _ParamsMappingValueType], - Mapping[int, _ParamsMappingValueType], - Mapping[float, _ParamsMappingValueType] + Mapping[Union[str, bytes, int, float], _ParamsMappingValueType], + Union[str, bytes], + Tuple[Union[str, bytes, int, float], _ParamsMappingValueType], + Mapping[str, _ParamsMappingValueType], + Mapping[bytes, _ParamsMappingValueType], + Mapping[int, _ParamsMappingValueType], + Mapping[float, _ParamsMappingValueType], ] ] Data = Union[ @@ -29,16 +41,17 @@ Data = Union[ MutableMapping[Text, str], MutableMapping[Text, Text], Iterable[Tuple[str, str]], - IO + IO, ] _Hook = Callable[[Response], Any] - Method = str URL = str Headers = Optional[Union[None, MutableMapping[Text, Text]]] Cookies = Optional[Union[None, RequestsCookieJar, MutableMapping[Text, Text]]] Files = Optional[MutableMapping[Text, IO]] -Auth = Union[None, Tuple[Text, Text], auth.AuthBase, Callable[[PreparedRequest], PreparedRequest]] +Auth = Union[ + None, Tuple[Text, Text], auth.AuthBase, Callable[[PreparedRequest], PreparedRequest] +] Timeout = Union[None, float, Tuple[float, float]] AllowRedirects = Optional[bool] Proxies = Optional[MutableMapping[Text, Text]] @@ -47,4 +60,4 @@ Stream = Optional[bool] Verify = Union[None, bool, Text] Cert = Union[Text, Tuple[Text, Text]] JSON = Optional[MutableMapping] -Help = Dict \ No newline at end of file +Help = Dict diff --git a/requests/utils.py b/requests/utils.py index 53988afb..1099fa74 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.utils ~~~~~~~~~~~~~~ @@ -21,38 +20,49 @@ import struct import warnings from .__version__ import __version__ -from . import certs +from .import certs + # to_native_string is unused here, but imported here for backwards compatibility from ._internal_utils import to_native_string from .basics import parse_http_list as _parse_list_header from .basics import ( - quote, urlparse, bytes, str, unquote, getproxies, - proxy_bypass, urlunparse, basestring, integer_types, - proxy_bypass_environment, getproxies_environment) + quote, + urlparse, + bytes, + str, + unquote, + getproxies, + proxy_bypass, + urlunparse, + basestring, + integer_types, + proxy_bypass_environment, + getproxies_environment, +) from .cookies import cookiejar_from_dict from .structures import CaseInsensitiveDict from .exceptions import ( - InvalidURL, InvalidHeader, FileModeWarning, UnrewindableBodyError) + InvalidURL, InvalidHeader, FileModeWarning, UnrewindableBodyError +) NETRC_FILES = ('.netrc', '_netrc') - DEFAULT_CA_BUNDLE_PATH = certs.where() - - if platform.system() == 'Windows': - # provide a proxy_bypass version on Windows without DNS lookups + # provide a proxy_bypass version on Windows without DNS lookups def proxy_bypass_registry(host): import winreg + try: - internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, - r'Software\Microsoft\Windows\CurrentVersion\Internet Settings') - proxyEnable = winreg.QueryValueEx(internetSettings, - 'ProxyEnable')[0] - proxyOverride = winreg.QueryValueEx(internetSettings, - 'ProxyOverride')[0] + internetSettings = winreg.OpenKey( + winreg.HKEY_CURRENT_USER, + r'Software\Microsoft\Windows\CurrentVersion\Internet Settings', + ) + proxyEnable = winreg.QueryValueEx(internetSettings, 'ProxyEnable')[0] + proxyOverride = winreg.QueryValueEx(internetSettings, 'ProxyOverride')[0] except OSError: return False + if not proxyEnable or not proxyOverride: return False @@ -65,11 +75,13 @@ if platform.system() == 'Windows': if test == '': if '.' not in host: return True - test = test.replace(".", r"\.") # mask dots - test = test.replace("*", r".*") # change glob sequence - test = test.replace("?", r".") # change glob char + + test = test.replace(".", r"\.") # mask dots + test = test.replace("*", r".*") # change glob sequence + test = test.replace("?", r".") # change glob char if re.match(test, host, re.I): return True + return False def proxy_bypass(host): # noqa @@ -80,29 +92,25 @@ if platform.system() == 'Windows': """ if getproxies_environment(): return proxy_bypass_environment(host) + else: return proxy_bypass_registry(host) def dict_to_sequence(d): """Returns an internal sequence dictionary update.""" - if hasattr(d, 'items'): d = d.items() - return d def super_len(o): total_length = None current_position = 0 - if hasattr(o, '__len__'): total_length = len(o) - elif hasattr(o, 'len'): total_length = o.len - elif hasattr(o, 'fileno'): try: fileno = o.fileno() @@ -110,20 +118,20 @@ def super_len(o): pass else: total_length = os.fstat(fileno).st_size - # Having used fstat to determine the file length, we need to # confirm that this file was opened up in binary mode. if 'b' not in o.mode: - warnings.warn(( - "Requests has determined the content-length for this " - "request using the binary size of the file: however, the " - "file has been opened in text mode (i.e. without the 'b' " - "flag in the mode). This may lead to an incorrect " - "content-length. In Requests 3.0, support will be removed " - "for files in text mode."), - FileModeWarning + warnings.warn( + ( + "Requests has determined the content-length for this " + "request using the binary size of the file: however, the " + "file has been opened in text mode (i.e. without the 'b' " + "flag in the mode). This may lead to an incorrect " + "content-length. In Requests 3.0, support will be removed " + "for files in text mode." + ), + FileModeWarning, ) - if hasattr(o, 'tell'): try: current_position = o.tell() @@ -141,27 +149,22 @@ def super_len(o): # seek to end of file o.seek(0, 2) total_length = o.tell() - # seek back to current position to support # partially read file-like objects o.seek(current_position or 0) except (OSError, IOError): total_length = 0 - if total_length is None: total_length = 0 - return max(0, total_length - current_position) def get_netrc_auth(url, raise_errors=False): """Returns the Requests tuple auth for a given url from netrc.""" - try: from netrc import netrc, NetrcParseError netrc_path = None - for f in NETRC_FILES: try: loc = os.path.expanduser('~/{0}'.format(f)) @@ -180,20 +183,19 @@ def get_netrc_auth(url, raise_errors=False): return ri = urlparse(url) - # Strip port numbers from netloc. This weird `if...encode`` dance is # used for Python 3.2, which doesn't support unicode literals. splitstr = b':' if isinstance(url, str): splitstr = splitstr.decode('ascii') host = ri.netloc.split(splitstr)[0] - try: _netrc = netrc(netrc_path).authenticators(host) if _netrc: # Return with login / password login_i = (0 if _netrc[0] else 1) return (_netrc[login_i], _netrc[2]) + except (NetrcParseError, IOError): # If there was a parsing error or a permissions issue reading the file, # we'll just skip netrc auth unless explicitly asked to raise errors. @@ -208,8 +210,7 @@ def get_netrc_auth(url, raise_errors=False): def guess_filename(obj): """Tries to guess the filename of the given object.""" name = getattr(obj, 'name', None) - if (name and isinstance(name, basestring) and name[0] != '<' and - name[-1] != '>'): + if (name and isinstance(name, basestring) and name[0] != '<' and name[-1] != '>'): return os.path.basename(name) @@ -261,10 +262,11 @@ def to_key_val_list(value): if isinstance(value, collections.Mapping): value = value.items() - return list(value) + + # From mitsuhiko/werkzeug (used with permission). def parse_list_header(value): """Parse lists as described by RFC 2068 Section 2. @@ -297,6 +299,8 @@ def parse_list_header(value): return result + + # From mitsuhiko/werkzeug (used with permission). def parse_dict_header(value): """Parse lists of key, value pairs as described by RFC 2068 Section 2 and @@ -325,6 +329,7 @@ def parse_dict_header(value): if '=' not in item: result[item] = None continue + name, value = item.split('=', 1) if value[:1] == value[-1:] == '"': value = unquote_header_value(value[1:-1]) @@ -332,6 +337,8 @@ def parse_dict_header(value): return result + + # From mitsuhiko/werkzeug (used with permission). def unquote_header_value(value, is_filename=False): r"""Unquotes a header value. (Reversal of :func:`quote_header_value`). @@ -347,7 +354,6 @@ def unquote_header_value(value, is_filename=False): # probably some other browsers as well. IE for example is # uploading files with "C:\foo\bar.txt" as filename value = value[1:-1] - # if this is a filename and the starting characters look like # a UNC path, then just return the value without quotes. Using the # replace sequence below on a UNC path has the effect of turning @@ -355,6 +361,7 @@ def unquote_header_value(value, is_filename=False): # _fix_ie_filename() doesn't work correctly. See #458. if not is_filename or value[:2] != '\\\\': return value.replace('\\\\', '\\').replace('\\"', '"') + return value @@ -364,12 +371,9 @@ def dict_from_cookiejar(cj): :param cj: CookieJar object to extract cookies from. :rtype: dict """ - cookie_dict = {} - for cookie in cj: cookie_dict[cookie.name] = cookie.value - return cookie_dict @@ -380,7 +384,6 @@ def add_dict_to_cookiejar(cj, cookie_dict): :param cookie_dict: Dict of key/values to insert into CookieJar. :rtype: CookieJar """ - return cookiejar_from_dict(cookie_dict, cj) @@ -389,19 +392,22 @@ def get_encodings_from_content(content): :param content: bytestring to extract encodings from. """ - warnings.warn(( - 'In requests 3.0, get_encodings_from_content will be removed. For ' - 'more information, please see the discussion on issue #2266. (This' - ' warning should only appear once.)'), - DeprecationWarning) - + warnings.warn( + ( + 'In requests 3.0, get_encodings_from_content will be removed. For ' + 'more information, please see the discussion on issue #2266. (This' + ' warning should only appear once.)' + ), + DeprecationWarning, + ) charset_re = re.compile(r']', flags=re.I) pragma_re = re.compile(r']', flags=re.I) xml_re = re.compile(r'^<\?xml.*?encoding=["\']*(.+?)["\'>]') - - return (charset_re.findall(content) + - pragma_re.findall(content) + - xml_re.findall(content)) + return ( + charset_re.findall(content) + + pragma_re.findall(content) + + xml_re.findall(content) + ) def get_encoding_from_headers(headers): @@ -410,14 +416,11 @@ def get_encoding_from_headers(headers): :param headers: dictionary to extract encoding from. :rtype: str """ - content_type = headers.get('content-type') - if not content_type: return None content_type, params = cgi.parse_header(content_type) - if 'charset' in params: return params['charset'].strip("'\"") @@ -427,12 +430,12 @@ def get_encoding_from_headers(headers): def stream_decode_response_unicode(iterator, r): """Stream decodes a iterator.""" - decoder = codecs.getincrementaldecoder(r.encoding)(errors='replace') for chunk in iterator: rv = decoder.decode(chunk) if rv: yield rv + rv = decoder.decode(b'', final=True) if rv: yield rv @@ -444,7 +447,8 @@ def iter_slices(string, slice_length): if slice_length is None or slice_length <= 0: slice_length = len(string) while pos < len(string): - yield string[pos:pos + slice_length] + yield string[pos: pos + slice_length] + pos += slice_length @@ -460,33 +464,35 @@ def get_unicode_from_response(r): :rtype: str """ - warnings.warn(( - 'In requests 3.0, get_unicode_from_response will be removed. For ' - 'more information, please see the discussion on issue #2266. (This' - ' warning should only appear once.)'), - DeprecationWarning) - + warnings.warn( + ( + 'In requests 3.0, get_unicode_from_response will be removed. For ' + 'more information, please see the discussion on issue #2266. (This' + ' warning should only appear once.)' + ), + DeprecationWarning, + ) tried_encodings = [] - # Try charset from content-type encoding = get_encoding_from_headers(r.headers) - if encoding: try: return str(r.content, encoding) + except UnicodeError: tried_encodings.append(encoding) - # Fall back: try: return str(r.content, encoding, errors='replace') + except TypeError: return r.content # The unreserved URI characters (RFC 3986) UNRESERVED_SET = frozenset( - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789-._~") + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789-._~" +) def unquote_unreserved(uri): @@ -495,12 +501,14 @@ def unquote_unreserved(uri): :rtype: str """ + # This convert function is used to optionally convert the output of `chr`. # In Python 3, `chr` returns a unicode string, while in Python 2 it returns # a bytestring. Here we deal with that by optionally converting. def convert(is_bytes, c): if is_bytes: return c.encode('ascii') + else: return c @@ -511,7 +519,6 @@ def unquote_unreserved(uri): if is_bytes: splitchar = splitchar.encode('ascii') base = base.encode('ascii') - parts = uri.split(splitchar) for i in range(1, len(parts)): h = parts[i][0:2] @@ -545,6 +552,7 @@ def requote_uri(uri): # Then quote only illegal characters (do not quote reserved, # unreserved, or '%') return quote(unquote_unreserved(uri), safe=safe_with_percent) + except InvalidURL: # We couldn't unquote the given URI, so let's try quoting it, but # there may be unquoted '%'s in the URI. We need to make sure they're @@ -564,7 +572,7 @@ def address_in_network(ip, net): netaddr, bits = net.split('/') netmask = struct.unpack('=L', socket.inet_aton(dotted_netmask(int(bits))))[0] network = struct.unpack('=L', socket.inet_aton(netaddr))[0] & netmask - return (ipaddr & netmask) == (network & netmask) + return ( ipaddr & netmask) == ( network & netmask) def dotted_netmask(mask): @@ -586,6 +594,7 @@ def is_ipv4_address(string_ip): socket.inet_aton(string_ip) except socket.error: return False + return True @@ -608,8 +617,10 @@ def is_valid_cidr(string_network): socket.inet_aton(string_network.split('/')[0]) except socket.error: return False + else: return False + return True @@ -627,6 +638,7 @@ def set_environ(env_name, value): os.environ[env_name] = value try: yield + finally: if value_changed: if old_value is None: @@ -642,31 +654,28 @@ def should_bypass_proxies(url, no_proxy): :rtype: bool """ get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper()) - # First check whether no_proxy is defined. If it is, check that the URL # we're getting isn't in the no_proxy list. no_proxy_arg = no_proxy if no_proxy is None: no_proxy = get_proxy('no_proxy') netloc = urlparse(url).netloc - if no_proxy: # We need to check whether we match here. We need to see if we match # the end of the netloc, both with and without the port. - no_proxy = ( - host for host in no_proxy.replace(' ', '').split(',') if host - ) - + no_proxy = (host for host in no_proxy.replace(' ', '').split(',') if host) ip = netloc.split(':')[0] if is_ipv4_address(ip): for proxy_ip in no_proxy: if is_valid_cidr(proxy_ip): if address_in_network(ip, proxy_ip): return True + elif ip == proxy_ip: # If no_proxy ip was defined in plain IP notation instead of cidr notation & # matches the IP of the index return True + else: for host in no_proxy: if netloc.endswith(host) or netloc.split(':')[0].endswith(host): @@ -686,6 +695,7 @@ def get_environ_proxies(url, no_proxy=None): """ if should_bypass_proxies(url, no_proxy=no_proxy): return {} + else: return getproxies() @@ -729,12 +739,14 @@ def default_headers(): """ :rtype: requests.structures.CaseInsensitiveDict """ - return CaseInsensitiveDict({ - 'User-Agent': default_user_agent(), - 'Accept-Encoding': ', '.join(('gzip', 'deflate')), - 'Accept': '*/*', - 'Connection': 'keep-alive', - }) + return CaseInsensitiveDict( + { + 'User-Agent': default_user_agent(), + 'Accept-Encoding': ', '.join(('gzip', 'deflate')), + 'Accept': '*/*', + 'Connection': 'keep-alive', + } + ) def parse_header_links(value): @@ -744,11 +756,8 @@ def parse_header_links(value): :rtype: list """ - links = [] - replace_chars = ' \'"' - value = value.strip(replace_chars) if not value: return links @@ -758,9 +767,7 @@ def parse_header_links(value): url, params = val.split(';', 1) except ValueError: url, params = val, '' - link = {'url': url.strip('<> \'"')} - for param in params.split(';'): try: key, value = param.split('=') @@ -768,9 +775,7 @@ def parse_header_links(value): break link[key.strip(replace_chars)] = value.strip(replace_chars) - links.append(link) - return links @@ -783,6 +788,7 @@ def is_valid_location(response): getlist = getattr(headers, 'getlist', None) if getlist is not None: return len(getlist('location')) <= 1 + # If response.raw isn't urllib3-like we can't reliably check this return True @@ -802,26 +808,34 @@ def guess_json_utf(data): # determine the encoding. Also detect a BOM, if present. sample = data[:4] if sample in (codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE): - return 'utf-32' # BOM included + return 'utf-32' # BOM included + if sample[:3] == codecs.BOM_UTF8: return 'utf-8-sig' # BOM included, MS style (discouraged) + if sample[:2] in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE): - return 'utf-16' # BOM included + return 'utf-16' # BOM included + nullcount = sample.count(_null) if nullcount == 0: return 'utf-8' + if nullcount == 2: - if sample[::2] == _null2: # 1st and 3rd are null + if sample[::2] == _null2: # 1st and 3rd are null return 'utf-16-be' + if sample[1::2] == _null2: # 2nd and 4th are null return 'utf-16-le' - # Did not detect 2 valid UTF-16 ascii-range characters + + # Did not detect 2 valid UTF-16 ascii-range characters if nullcount == 3: if sample[:3] == _null3: return 'utf-32-be' + if sample[1:] == _null3: return 'utf-32-le' - # Did not detect a valid UTF-32 ascii-range character + + # Did not detect a valid UTF-32 ascii-range character return None @@ -832,13 +846,11 @@ def prepend_scheme_if_needed(url, new_scheme): :rtype: str """ scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme) - # urlparse is a finicky beast, and sometimes decides that there isn't a # netloc present. Assume that it's being over-cautious, and switch netloc # and path if urlparse decided there was no netloc. if not netloc: netloc, path = path, netloc - return urlunparse((scheme, netloc, path, params, query, fragment)) @@ -849,12 +861,10 @@ def get_auth_from_url(url): :rtype: (str,str) """ parsed = urlparse(url) - try: auth = (unquote(parsed.username), unquote(parsed.password)) except (AttributeError, TypeError): auth = ('', '') - return auth @@ -871,17 +881,21 @@ def check_header_validity(header): :param header: tuple, in the format (name, value). """ name, value = header - if isinstance(value, bytes): pat = _CLEAN_HEADER_REGEX_BYTE else: pat = _CLEAN_HEADER_REGEX_STR try: if not pat.match(value): - raise InvalidHeader("Invalid return character or leading space in header: %s" % name) + raise InvalidHeader( + "Invalid return character or leading space in header: %s" % name + ) + except TypeError: - raise InvalidHeader("Value for header {%s: %s} must be of type str or " - "bytes, not %s" % (name, value, type(value))) + raise InvalidHeader( + "Value for header {%s: %s} must be of type str or " + "bytes, not %s" % (name, value, type(value)) + ) def urldefragauth(url): @@ -891,13 +905,10 @@ def urldefragauth(url): :rtype: str """ scheme, netloc, path, params, query, fragment = urlparse(url) - # see func:`prepend_scheme_if_needed` if not netloc: netloc, path = path, netloc - netloc = netloc.rsplit('@', 1)[-1] - return urlunparse((scheme, netloc, path, params, query, '')) @@ -906,12 +917,16 @@ def rewind_body(prepared_request): so it can be read again on redirect. """ body_seek = getattr(prepared_request.body, 'seek', None) - if body_seek is not None and isinstance(prepared_request._body_position, integer_types): + if body_seek is not None and isinstance( + prepared_request._body_position, integer_types + ): try: body_seek(prepared_request._body_position) except (IOError, OSError): - raise UnrewindableBodyError("An error occurred when rewinding request " - "body for redirect.") + raise UnrewindableBodyError( + "An error occurred when rewinding request " "body for redirect." + ) + else: raise UnrewindableBodyError("Unable to rewind request body for redirect.") diff --git a/setup.py b/setup.py index cf329a9f..e7bbd228 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ #!/usr/bin/env python # Learn more: https://github.com/kennethreitz/setup.py - import os import sys @@ -55,27 +54,30 @@ if sys.argv[-1] == 'publish': os.system('python setup.py sdist bdist_wheel') os.system('twine upload dist/*') sys.exit() - packages = ['requests'] - requires = [ 'chardet>=3.0.2,<3.1.0', 'idna>=2.5,<2.7', 'urllib3>=1.21.1,<1.23', - 'certifi>=2017.4.17' - + 'certifi>=2017.4.17', +] +test_requirements = [ + 'pytest-httpbin==0.0.7', + 'pytest-cov', + 'pytest-mock', + 'pytest-xdist', + 'PySocks>=1.5.6, !=1.5.7', + 'pytest>=2.8.0', + 'pytest-mypy', + 'mypy==0.540', ] -test_requirements = ['pytest-httpbin==0.0.7', 'pytest-cov', 'pytest-mock', 'pytest-xdist', 'PySocks>=1.5.6, !=1.5.7', 'pytest>=2.8.0', 'pytest-mypy', 'mypy==0.540'] - about = {} with open(os.path.join(here, 'requests', '__version__.py'), 'r', 'utf-8') as f: - exec(f.read(), about) - + exec (f.read(), about) with open('README.rst', 'r', 'utf-8') as f: readme = f.read() with open('HISTORY.rst', 'r', 'utf-8') as f: history = f.read() - setup( name=about['__title__'], version=about['__version__'], @@ -102,12 +104,9 @@ setup( 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy' + 'Programming Language :: Python :: Implementation :: PyPy', ), - cmdclass={ - 'test': PyTest, - 'mypy': MyPyTest - }, + cmdclass={'test': PyTest, 'mypy': MyPyTest}, tests_require=test_requirements, extras_require={ 'security': ['pyOpenSSL>=0.14', 'cryptography>=1.3.4', 'idna>=2.0.0'], diff --git a/tests/__init__.py b/tests/__init__.py index 9be94bcc..bb676233 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """Requests test package initialisation.""" import warnings diff --git a/tests/compat.py b/tests/compat.py index ad51a1b3..506331e2 100644 --- a/tests/compat.py +++ b/tests/compat.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- - import io as StringIO + def u(s): return s diff --git a/tests/conftest.py b/tests/conftest.py index 6aa8edfe..79486a30 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - import pytest from urllib.parse import urljoin diff --git a/tests/test_help.py b/tests/test_help.py index c11d43f3..a61c9ebd 100644 --- a/tests/test_help.py +++ b/tests/test_help.py @@ -1,5 +1,4 @@ # -*- encoding: utf-8 - import sys import pytest @@ -7,7 +6,7 @@ import pytest from requests.help import info -@pytest.mark.skipif(sys.version_info[:2] != (2,6), reason="Only run on Python 2.6") +@pytest.mark.skipif(sys.version_info[:2] != (2, 6), reason="Only run on Python 2.6") def test_system_ssl_py26(): """OPENSSL_VERSION_NUMBER isn't provided in Python 2.6, verify we don't blow up in this case. @@ -15,13 +14,14 @@ def test_system_ssl_py26(): assert info()['system_ssl'] == {'version': ''} -@pytest.mark.skipif(sys.version_info < (2,7), reason="Only run on Python 2.7+") +@pytest.mark.skipif(sys.version_info < (2, 7), reason="Only run on Python 2.7+") def test_system_ssl(): """Verify we're actually setting system_ssl when it should be available.""" assert info()['system_ssl']['version'] != '' class VersionedPackage(object): + def __init__(self, version): self.__version__ = version diff --git a/tests/test_hooks.py b/tests/test_hooks.py index 014b4391..126ec97d 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - import pytest from requests import hooks @@ -10,10 +9,7 @@ def hook(value): @pytest.mark.parametrize( - 'hooks_list, result', ( - (hook, 'ata'), - ([hook, lambda x: None, hook], 'ta'), - ) + 'hooks_list, result', ((hook, 'ata'), ([hook, lambda x: None, hook], 'ta')) ) def test_hooks(hooks_list, result): assert hooks.dispatch_hook('response', {'response': hooks_list}, 'Data') == result diff --git a/tests/test_lowlevel.py b/tests/test_lowlevel.py index 4161f875..c02f2b8f 100644 --- a/tests/test_lowlevel.py +++ b/tests/test_lowlevel.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - import pytest import threading import requests @@ -14,23 +13,20 @@ def test_chunked_upload(): close_server = threading.Event() server = Server.basic_response_server(wait_to_close_event=close_server) data = iter([b'a', b'b', b'c']) - with server as (host, port): url = 'http://{0}:{1}/'.format(host, port) r = requests.post(url, data=data, stream=True) close_server.set() # release server block - assert r.status_code == 200 assert r.request.headers['Transfer-Encoding'] == 'chunked' + def test_incorrect_content_length(): """Test ConnectionError raised for incomplete responses""" close_server = threading.Event() server = Server.text_response_server( - "HTTP/1.1 200 OK\r\n" + - "Content-Length: 50\r\n\r\n" + - "Hello World." - ) + "HTTP/1.1 200 OK\r\n" + "Content-Length: 50\r\n\r\n" + "Hello World." + ) with server as (host, port): url = 'http://{0}:{1}/'.format(host, port) r = requests.Request('GET', url).prepare() @@ -47,23 +43,22 @@ def test_digestauth_401_count_reset_on_redirect(): See https://github.com/requests/requests/issues/1979. """ - text_401 = (b'HTTP/1.1 401 UNAUTHORIZED\r\n' - b'Content-Length: 0\r\n' - b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"' - b', opaque="372825293d1c26955496c80ed6426e9e", ' - b'realm="me@kennethreitz.com", qop=auth\r\n\r\n') - - text_302 = (b'HTTP/1.1 302 FOUND\r\n' - b'Content-Length: 0\r\n' - b'Location: /\r\n\r\n') - - text_200 = (b'HTTP/1.1 200 OK\r\n' - b'Content-Length: 0\r\n\r\n') - - expected_digest = (b'Authorization: Digest username="user", ' - b'realm="me@kennethreitz.com", ' - b'nonce="6bf5d6e4da1ce66918800195d6b9130d", uri="/"') - + text_401 = ( + b'HTTP/1.1 401 UNAUTHORIZED\r\n' + b'Content-Length: 0\r\n' + b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"' + b', opaque="372825293d1c26955496c80ed6426e9e", ' + b'realm="me@kennethreitz.com", qop=auth\r\n\r\n' + ) + text_302 = ( + b'HTTP/1.1 302 FOUND\r\n' b'Content-Length: 0\r\n' b'Location: /\r\n\r\n' + ) + text_200 = (b'HTTP/1.1 200 OK\r\n' b'Content-Length: 0\r\n\r\n') + expected_digest = ( + b'Authorization: Digest username="user", ' + b'realm="me@kennethreitz.com", ' + b'nonce="6bf5d6e4da1ce66918800195d6b9130d", uri="/"' + ) auth = requests.auth.HTTPDigestAuth('user', 'pass') def digest_response_handler(sock): @@ -71,28 +66,23 @@ def test_digestauth_401_count_reset_on_redirect(): request_content = consume_socket_content(sock, timeout=0.5) assert request_content.startswith(b"GET / HTTP/1.1") sock.send(text_401) - # Verify we receive an Authorization header in response, then redirect. request_content = consume_socket_content(sock, timeout=0.5) assert expected_digest in request_content sock.send(text_302) - # Verify Authorization isn't sent to the redirected host, # then send another challenge. request_content = consume_socket_content(sock, timeout=0.5) assert b'Authorization:' not in request_content sock.send(text_401) - # Verify Authorization is sent correctly again, and return 200 OK. request_content = consume_socket_content(sock, timeout=0.5) assert expected_digest in request_content sock.send(text_200) - return request_content close_server = threading.Event() server = Server(digest_response_handler, wait_to_close_event=close_server) - with server as (host, port): url = 'http://{0}:{1}/'.format(host, port) r = requests.get(url, auth=auth) @@ -110,16 +100,18 @@ def test_digestauth_401_only_sent_once(): """Ensure we correctly respond to a 401 challenge once, and then stop responding if challenged again. """ - text_401 = (b'HTTP/1.1 401 UNAUTHORIZED\r\n' - b'Content-Length: 0\r\n' - b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"' - b', opaque="372825293d1c26955496c80ed6426e9e", ' - b'realm="me@kennethreitz.com", qop=auth\r\n\r\n') - - expected_digest = (b'Authorization: Digest username="user", ' - b'realm="me@kennethreitz.com", ' - b'nonce="6bf5d6e4da1ce66918800195d6b9130d", uri="/"') - + text_401 = ( + b'HTTP/1.1 401 UNAUTHORIZED\r\n' + b'Content-Length: 0\r\n' + b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"' + b', opaque="372825293d1c26955496c80ed6426e9e", ' + b'realm="me@kennethreitz.com", qop=auth\r\n\r\n' + ) + expected_digest = ( + b'Authorization: Digest username="user", ' + b'realm="me@kennethreitz.com", ' + b'nonce="6bf5d6e4da1ce66918800195d6b9130d", uri="/"' + ) auth = requests.auth.HTTPDigestAuth('user', 'pass') def digest_failed_response_handler(sock): @@ -127,22 +119,18 @@ def test_digestauth_401_only_sent_once(): request_content = consume_socket_content(sock, timeout=0.5) assert request_content.startswith(b"GET / HTTP/1.1") sock.send(text_401) - # Verify we receive an Authorization header in response, then # challenge again. request_content = consume_socket_content(sock, timeout=0.5) assert expected_digest in request_content sock.send(text_401) - # Verify the client didn't respond to second challenge. request_content = consume_socket_content(sock, timeout=0.5) assert request_content == b'' - return request_content close_server = threading.Event() server = Server(digest_failed_response_handler, wait_to_close_event=close_server) - with server as (host, port): url = 'http://{0}:{1}/'.format(host, port) r = requests.get(url, auth=auth) @@ -157,12 +145,13 @@ def test_digestauth_only_on_4xx(): See https://github.com/requests/requests/issues/3772. """ - text_200_chal = (b'HTTP/1.1 200 OK\r\n' - b'Content-Length: 0\r\n' - b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"' - b', opaque="372825293d1c26955496c80ed6426e9e", ' - b'realm="me@kennethreitz.com", qop=auth\r\n\r\n') - + text_200_chal = ( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 0\r\n' + b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"' + b', opaque="372825293d1c26955496c80ed6426e9e", ' + b'realm="me@kennethreitz.com", qop=auth\r\n\r\n' + ) auth = requests.auth.HTTPDigestAuth('user', 'pass') def digest_response_handler(sock): @@ -170,16 +159,13 @@ def test_digestauth_only_on_4xx(): request_content = consume_socket_content(sock, timeout=0.5) assert request_content.startswith(b"GET / HTTP/1.1") sock.send(text_200_chal) - # Verify the client didn't respond with auth. request_content = consume_socket_content(sock, timeout=0.5) assert request_content == b'' - return request_content close_server = threading.Event() server = Server(digest_response_handler, wait_to_close_event=close_server) - with server as (host, port): url = 'http://{0}:{1}/'.format(host, port) r = requests.get(url, auth=auth) @@ -190,16 +176,12 @@ def test_digestauth_only_on_4xx(): _schemes_by_var_prefix = [ - ('http', ['http']), - ('https', ['https']), - ('all', ['http', 'https']), + ('http', ['http']), ('https', ['https']), ('all', ['http', 'https']) ] - _proxy_combos = [] for prefix, schemes in _schemes_by_var_prefix: for scheme in schemes: _proxy_combos.append(("{0}_proxy".format(prefix), scheme)) - _proxy_combos += [(var.upper(), scheme) for var, scheme in _proxy_combos] @@ -214,10 +196,8 @@ def test_use_proxy_from_environment(httpbin, var, scheme): # fake proxy's lack of response will cause a ConnectionError with pytest.raises(requests.exceptions.ConnectionError): requests.get(url) - # the fake proxy received a request assert len(fake_proxy.handler_results) == 1 - # it had actual content (not checking for SOCKS protocol for now) assert len(fake_proxy.handler_results[0]) > 0 @@ -241,7 +221,6 @@ def test_redirect_rfc1808_to_non_ascii_location(): close_server = threading.Event() server = Server(redirect_resp_handler, wait_to_close_event=close_server) - with server as (host, port): url = u'http://{0}:{1}'.format(host, port) r = requests.get(url=url, allow_redirects=True) @@ -250,5 +229,4 @@ def test_redirect_rfc1808_to_non_ascii_location(): assert r.history[0].status_code == 301 assert redirect_request[0].startswith(b'GET /' + expected_path + b' HTTP/1.1') assert r.url == u'{0}/{1}'.format(url, expected_path.decode('ascii')) - close_server.set() diff --git a/tests/test_requests.py b/tests/test_requests.py index e323d83b..5e5ae666 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """Tests for Requests.""" from __future__ import division @@ -16,16 +15,24 @@ import pytest import pytest_httpbin from requests.adapters import HTTPAdapter from requests.auth import HTTPDigestAuth, _basic_auth_str -from requests.basics import ( - Morsel, cookielib, getproxies, str, urlparse, - builtin_str) -from requests.cookies import ( - cookiejar_from_dict, morsel_to_cookie) +from requests.basics import ( Morsel, cookielib, getproxies, str, urlparse, builtin_str) +from requests.cookies import ( cookiejar_from_dict, morsel_to_cookie) from requests.exceptions import ( - ConnectionError, ConnectTimeout, InvalidScheme, InvalidURL, - MissingScheme, ReadTimeout, Timeout, RetryError, TooManyRedirects, - ProxyError, InvalidHeader, UnrewindableBodyError, InvalidBodyError, - SSLError) + ConnectionError, + ConnectTimeout, + InvalidScheme, + InvalidURL, + MissingScheme, + ReadTimeout, + Timeout, + RetryError, + TooManyRedirects, + ProxyError, + InvalidHeader, + UnrewindableBodyError, + InvalidBodyError, + SSLError, +) from requests.models import PreparedRequest from requests.structures import CaseInsensitiveDict from requests.sessions import SessionRedirectMixin @@ -37,14 +44,15 @@ from .compat import StringIO, u from .utils import override_environ from urllib3.util import Timeout as Urllib3Timeout + class SendRecordingAdapter(HTTPAdapter): """ A basic subclass of the HTTPAdapter that records the arguments used to ``send``. """ + def __init__(self, *args, **kwargs): super(SendRecordingAdapter, self).__init__(*args, **kwargs) - self.send_calls = [] def send(self, *args, **kwargs): @@ -55,14 +63,13 @@ class SendRecordingAdapter(HTTPAdapter): # Requests to this URL should always fail with a connection timeout (nothing # listening on that port) TARPIT = 'http://10.255.255.1' - try: from ssl import SSLContext + del SSLContext HAS_MODERN_SSL = True except ImportError: HAS_MODERN_SSL = False - try: requests.pyopenssl HAS_PYOPENSSL = True @@ -73,7 +80,6 @@ except AttributeError: class TestRequests: def test_entry_points(self): - requests.session requests.session().get requests.session().head @@ -84,13 +90,15 @@ class TestRequests: requests.post @pytest.mark.parametrize( - 'exception, url', ( + 'exception, url', + ( (MissingScheme, 'hiwpefhipowhefopw'), (InvalidScheme, 'localhost:3128'), (InvalidScheme, 'localhost.localdomain:3128/'), (InvalidScheme, '10.122.1.1:3128/'), (InvalidURL, 'http://'), - )) + ), + ) def test_invalid_url(self, exception, url): with pytest.raises(exception): requests.get(url) @@ -99,7 +107,6 @@ class TestRequests: req = requests.Request(method='GET') req.url = 'http://kennethreitz.org/' req.data = {'life': '42'} - pr = req.prepare() assert pr.url == req.url assert pr.body == 'life=42' @@ -120,43 +127,55 @@ class TestRequests: assert req.headers['Content-Length'] == '0' def test_override_content_length(self, httpbin): - headers = { - 'Content-Length': 'not zero' - } + headers = {'Content-Length': 'not zero'} r = requests.Request('POST', httpbin('post'), headers=headers).prepare() assert 'Content-Length' in r.headers assert r.headers['Content-Length'] == 'not zero' def test_path_is_not_double_encoded(self): request = requests.Request('GET', "http://0.0.0.0/get/test case").prepare() - assert request.path_url == '/get/test%20case' @pytest.mark.parametrize( - 'url, expected', ( - ('http://example.com/path#fragment', 'http://example.com/path?a=b#fragment'), - ('http://example.com/path?key=value#fragment', 'http://example.com/path?key=value&a=b#fragment') - )) + 'url, expected', + ( + ( + 'http://example.com/path#fragment', + 'http://example.com/path?a=b#fragment', + ), + ( + 'http://example.com/path?key=value#fragment', + 'http://example.com/path?key=value&a=b#fragment', + ), + ), + ) def test_params_are_added_before_fragment(self, url, expected): request = requests.Request('GET', url, params={"a": "b"}).prepare() assert request.url == expected def test_params_original_order_is_preserved_by_default(self): param_ordered_dict = collections.OrderedDict( - (('z', 1), ('a', 1), ('k', 1), ('d', 1))) + (('z', 1), ('a', 1), ('k', 1), ('d', 1)) + ) session = requests.Session() - request = requests.Request('GET', 'http://example.com/', params=param_ordered_dict) + request = requests.Request( + 'GET', 'http://example.com/', params=param_ordered_dict + ) prep = session.prepare_request(request) assert prep.url == 'http://example.com/?z=1&a=1&k=1&d=1' def test_params_bytes_are_encoded(self): - request = requests.Request('GET', 'http://example.com', - params=b'test=foo').prepare() + request = requests.Request( + 'GET', 'http://example.com', params=b'test=foo' + ).prepare( + ) assert request.url == 'http://example.com/?test=foo' def test_binary_put(self): - request = requests.Request('PUT', 'http://example.com', - data=u"ööö".encode("utf-8")).prepare() + request = requests.Request( + 'PUT', 'http://example.com', data=u"ööö".encode("utf-8") + ).prepare( + ) assert isinstance(request.body, bytes) def test_whitespaces_are_removed_from_url(self): @@ -178,9 +197,7 @@ class TestRequests: r = requests.Request('GET', httpbin('get')) s = requests.Session() s.proxies = getproxies() - r = s.send(r.prepare()) - assert r.status_code == 200 def test_HTTP_302_ALLOW_REDIRECT_GET(self, httpbin): @@ -190,7 +207,11 @@ class TestRequests: assert r.history[0].is_redirect def test_HTTP_307_ALLOW_REDIRECT_POST(self, httpbin): - r = requests.post(httpbin('redirect-to'), data='test', params={'url': 'post', 'status_code': 307}) + r = requests.post( + httpbin('redirect-to'), + data='test', + params={'url': 'post', 'status_code': 307}, + ) assert r.status_code == 200 assert r.history[0].status_code == 307 assert r.history[0].is_redirect @@ -198,7 +219,11 @@ class TestRequests: def test_HTTP_307_ALLOW_REDIRECT_POST_WITH_SEEKABLE(self, httpbin): byte_str = b'test' - r = requests.post(httpbin('redirect-to'), data=io.BytesIO(byte_str), params={'url': 'post', 'status_code': 307}) + r = requests.post( + httpbin('redirect-to'), + data=io.BytesIO(byte_str), + params={'url': 'post', 'status_code': 307}, + ) assert r.status_code == 200 assert r.history[0].status_code == 307 assert r.history[0].is_redirect @@ -226,17 +251,20 @@ class TestRequests: assert e.response.url == url assert len(e.response.history) == 5 else: - pytest.fail('Expected custom max number of redirects to be respected but was not') + pytest.fail( + 'Expected custom max number of redirects to be respected but was not' + ) @pytest.mark.parametrize( - 'method, body, expected', ( + 'method, body, expected', + ( ('GET', None, 'GET'), ('HEAD', None, 'HEAD'), ('POST', 'test', 'GET'), ('PUT', 'put test', 'PUT'), ('PATCH', 'patch test', 'PATCH'), - ('DELETE', '', 'DELETE') - ) + ('DELETE', '', 'DELETE'), + ), ) def test_http_301_for_redirectable_methods(self, httpbin, method, body, expected): """Tests all methods except OPTIONS for expected redirect behaviour. @@ -247,26 +275,25 @@ class TestRequests: """ params = {'url': '/%s' % expected.lower(), 'status_code': '301'} r = requests.request(method, httpbin('redirect-to'), data=body, params=params) - assert r.request.url == httpbin(expected.lower()) assert r.request.method == expected assert r.history[0].status_code == 301 assert r.history[0].is_redirect - if expected in ('GET', 'HEAD'): assert r.request.body is None else: assert r.json()['data'] == body @pytest.mark.parametrize( - 'method, body, expected', ( + 'method, body, expected', + ( ('GET', None, 'GET'), ('HEAD', None, 'HEAD'), ('POST', 'test', 'GET'), ('PUT', 'put test', 'PUT'), ('PATCH', 'patch test', 'PATCH'), - ('DELETE', '', 'DELETE') - ) + ('DELETE', '', 'DELETE'), + ), ) def test_http_302_for_redirectable_methods(self, httpbin, method, body, expected): """Tests all methods except OPTIONS for expected redirect behaviour. @@ -277,26 +304,25 @@ class TestRequests: """ params = {'url': '/%s' % expected.lower()} r = requests.request(method, httpbin('redirect-to'), data=body, params=params) - assert r.request.url == httpbin(expected.lower()) assert r.request.method == expected assert r.history[0].status_code == 302 assert r.history[0].is_redirect - if expected in ('GET', 'HEAD'): assert r.request.body is None else: assert r.json()['data'] == body @pytest.mark.parametrize( - 'method, body, expected', ( + 'method, body, expected', + ( ('GET', None, 'GET'), ('HEAD', None, 'HEAD'), ('POST', 'test', 'GET'), ('PUT', 'put test', 'GET'), ('PATCH', 'patch test', 'GET'), - ('DELETE', '', 'GET') - ) + ('DELETE', '', 'GET'), + ), ) def test_http_303_for_redirectable_methods(self, httpbin, method, body, expected): """Tests all methods except OPTIONS for expected redirect behaviour. @@ -307,17 +333,16 @@ class TestRequests: """ params = {'url': '/%s' % expected.lower(), 'status_code': '303'} r = requests.request(method, httpbin('redirect-to'), data=body, params=params) - assert r.request.url == httpbin(expected.lower()) assert r.request.method == expected assert r.history[0].status_code == 303 assert r.history[0].is_redirect - assert r.request.body is None def test_multiple_location_headers(self, httpbin): - headers = [('Location', 'http://example.com'), - ('Location', 'https://example.com/1')] + headers = [ + ('Location', 'http://example.com'), ('Location', 'https://example.com/1') + ] params = '&'.join(['%s=%s' % (k, v) for k, v in headers]) ses = requests.Session() req = requests.Request('GET', httpbin('response-headers?%s' % params)) @@ -335,11 +360,9 @@ class TestRequests: req = requests.Request('POST', httpbin('post'), data={'test': 'data'}) prep = ses.prepare_request(req) resp = ses.send(prep) - # Mimic a redirect response resp.status_code = 302 resp.headers['location'] = 'get' - # Run request through resolve_redirects next_resp = next(ses.resolve_redirects(resp, prep)) assert next_resp.request.body is None @@ -352,17 +375,14 @@ class TestRequests: req = requests.Request('POST', httpbin('post'), data=(b'x' for x in range(1))) prep = ses.prepare_request(req) assert 'Transfer-Encoding' in prep.headers - # Create Response to avoid https://github.com/kevin1024/pytest-httpbin/issues/33 resp = requests.Response() resp.raw = io.BytesIO(b'the content') resp.request = prep setattr(resp.raw, 'release_conn', lambda *args: args) - # Mimic a redirect response resp.status_code = 302 resp.headers['location'] = httpbin('get') - # Run request through resolve_redirect next_resp = next(ses.resolve_redirects(resp, prep)) assert next_resp.request.body is None @@ -371,16 +391,15 @@ class TestRequests: def test_HTTP_200_OK_GET_WITH_PARAMS(self, httpbin): heads = {'User-agent': 'Mozilla/5.0'} - r = requests.get(httpbin('user-agent'), headers=heads) - assert heads['User-agent'] in r.text assert r.status_code == 200 def test_HTTP_200_OK_GET_WITH_MIXED_PARAMS(self, httpbin): heads = {'User-agent': 'Mozilla/5.0'} - - r = requests.get(httpbin('get') + '?test=true', params={'q': 'test'}, headers=heads) + r = requests.get( + httpbin('get') + '?test=true', params={'q': 'test'}, headers=heads + ) assert r.status_code == 200 def test_set_cookie_on_301(self, httpbin): @@ -401,10 +420,7 @@ class TestRequests: assert s.cookies['foo'] == 'bar' s.get( httpbin('response-headers'), - params={ - 'Set-Cookie': - 'foo=deleted; expires=Thu, 01-Jan-1970 00:00:01 GMT' - } + params={'Set-Cookie': 'foo=deleted; expires=Thu, 01-Jan-1970 00:00:01 GMT'}, ) assert 'foo' not in s.cookies @@ -461,23 +477,19 @@ class TestRequests: cj = cookiejar_from_dict({'foo': 'bar'}, cookielib.CookieJar()) s = requests.Session() s.cookies = cookiejar_from_dict({'cookie': 'tasty'}) - # Prepare request without using Session req = requests.Request('GET', httpbin('headers'), cookies=cj) prep_req = req.prepare() - # Send request and simulate redirect resp = s.send(prep_req) resp.status_code = 302 resp.headers['location'] = httpbin('get') redirects = s.resolve_redirects(resp, prep_req) resp = next(redirects) - # Verify CookieJar isn't being converted to RequestsCookieJar assert isinstance(prep_req._cookies, cookielib.CookieJar) assert isinstance(resp.request._cookies, cookielib.CookieJar) assert not isinstance(resp.request._cookies, requests.cookies.RequestsCookieJar) - cookies = {} for c in resp.request._cookies: cookies[c.name] = c.value @@ -485,28 +497,27 @@ class TestRequests: assert cookies['cookie'] == 'tasty' @pytest.mark.parametrize( - 'jar', ( - requests.cookies.RequestsCookieJar(), - cookielib.CookieJar() - )) + 'jar', (requests.cookies.RequestsCookieJar(), cookielib.CookieJar()) + ) def test_custom_cookie_policy_persistence(self, httpbin, jar): """Verify a custom CookiePolicy is propagated on each session request.""" class TestCookiePolicy(cookielib.DefaultCookiePolicy): """Policy to restrict all cookies from localhost (127.0.0.1).""" + def __init__(self): - cookielib.DefaultCookiePolicy.__init__(self, blocked_domains=['127.0.0.1']) + cookielib.DefaultCookiePolicy.__init__( + self, blocked_domains=['127.0.0.1'] + ) # Establish session with jar and set some cookies. s = requests.Session() s.cookies = jar s.get(httpbin('cookies/set?k1=v1&k2=v2')) assert len(s.cookies) == 2 - # Set different policy. s.cookies.set_policy(TestCookiePolicy()) assert isinstance(s.cookies._policy, TestCookiePolicy) - # No cookies were sent to our blocked domain and none were set. resp = s.get(httpbin('cookies/set?k3=v3')) assert 'Cookie' not in resp.request.headers @@ -557,9 +568,7 @@ class TestRequests: @pytest.mark.parametrize('key', ('User-agent', 'user-agent')) def test_user_agent_transfers(self, httpbin, key): - heads = {key: 'Mozilla/5.0 (github.com/requests/requests)'} - r = requests.get(httpbin('user-agent'), headers=heads) assert heads[key] in r.text @@ -574,42 +583,31 @@ class TestRequests: def test_BASICAUTH_TUPLE_HTTP_200_OK_GET(self, httpbin): auth = ('user', 'pass') url = httpbin('basic-auth', 'user', 'pass') - r = requests.get(url, auth=auth) assert r.status_code == 200 - r = requests.get(url) assert r.status_code == 401 - s = requests.session() s.auth = auth r = s.get(url) assert r.status_code == 200 @pytest.mark.parametrize( - 'username, password', ( - ('user', 'pass'), - (u'имя'.encode('utf-8'), u'пароль'.encode('utf-8')), - )) + 'username, password', + (('user', 'pass'), (u'имя'.encode('utf-8'), u'пароль'.encode('utf-8'))), + ) def test_set_basicauth(self, httpbin, username, password): auth = (username, password) url = httpbin('get') - r = requests.Request('GET', url, auth=auth) p = r.prepare() - assert p.headers['Authorization'] == _basic_auth_str(username, password) - @pytest.mark.parametrize( - 'username, password', ( - ('user', 1234), - (None, 'test'), - )) + @pytest.mark.parametrize('username, password', (('user', 1234), (None, 'test'))) def test_non_str_basicauth(self, username, password): """Ensure we only allow string or bytes values for basicauth""" with pytest.raises(TypeError) as e: requests.auth._basic_auth_str(username, password) - assert 'must be of type str or bytes' in str(e) def test_basicauth_encodes_byte_strings(self): @@ -619,18 +617,15 @@ class TestRequests: auth = (b'\xc5\xafsername', b'test\xc6\xb6') r = requests.Request('GET', 'http://localhost', auth=auth) p = r.prepare() - assert p.headers['Authorization'] == 'Basic xa9zZXJuYW1lOnRlc3TGtg==' @pytest.mark.parametrize( - 'url, exception', ( - # Connecting to an unknown domain should raise a ConnectionError - ('http://doesnotexist.google.com', ConnectionError), - # Connecting to an invalid port should raise a ConnectionError - ('http://localhost:1', ConnectionError), - # Inputing a URL that cannot be parsed should raise an InvalidURL error - ('http://fe80::5054:ff:fe5a:fc0', InvalidURL) - )) + 'url, exception', + (('http://doesnotexist.google.com', ConnectionError), ('http://localhost:1', ConnectionError), ('http://fe80::5054:ff:fe5a:fc0', InvalidURL)), + # Connecting to an unknown domain should raise a ConnectionError + # Connecting to an invalid port should raise a ConnectionError + # Inputing a URL that cannot be parsed should raise an InvalidURL error + ) def test_errors(self, url, exception): with pytest.raises(exception): requests.get(url, timeout=1) @@ -638,34 +633,31 @@ class TestRequests: def test_proxy_error(self): # any proxy related error (address resolution, no route to host, etc) should result in a ProxyError with pytest.raises(ProxyError): - requests.get('http://localhost:1', proxies={'http': 'non-resolvable-address'}) + requests.get( + 'http://localhost:1', proxies={'http': 'non-resolvable-address'} + ) def test_basicauth_with_netrc(self, httpbin): auth = ('user', 'pass') wrong_auth = ('wronguser', 'wrongpass') url = httpbin('basic-auth', 'user', 'pass') - old_auth = requests.sessions.get_netrc_auth - try: + def get_netrc_auth_mock(url): return auth - requests.sessions.get_netrc_auth = get_netrc_auth_mock + requests.sessions.get_netrc_auth = get_netrc_auth_mock # Should use netrc and work. r = requests.get(url) assert r.status_code == 200 - # Given auth should override and fail. r = requests.get(url, auth=wrong_auth) assert r.status_code == 401 - s = requests.session() - # Should use netrc and work. r = s.get(url) assert r.status_code == 200 - # Given auth should override and fail. s.auth = wrong_auth r = s.get(url) @@ -674,16 +666,12 @@ class TestRequests: requests.sessions.get_netrc_auth = old_auth def test_DIGEST_HTTP_200_OK_GET(self, httpbin): - auth = HTTPDigestAuth('user', 'pass') url = httpbin('digest-auth', 'auth', 'user', 'pass') - r = requests.get(url, auth=auth) assert r.status_code == 200 - r = requests.get(url) assert r.status_code == 401 - s = requests.session() s.auth = HTTPDigestAuth('user', 'pass') r = s.get(url) @@ -694,7 +682,6 @@ class TestRequests: auth = HTTPDigestAuth('user', 'pass') r = requests.get(url) assert r.cookies['fake'] == 'fake_value' - r = requests.get(url, auth=auth) assert r.status_code == 200 @@ -706,61 +693,48 @@ class TestRequests: assert s.cookies['fake'] == 'fake_value' def test_DIGEST_STREAM(self, httpbin): - auth = HTTPDigestAuth('user', 'pass') url = httpbin('digest-auth', 'auth', 'user', 'pass') - r = requests.get(url, auth=auth, stream=True) assert r.raw.read() != b'' - r = requests.get(url, auth=auth, stream=False) assert r.raw.read() == b'' def test_DIGESTAUTH_WRONG_HTTP_401_GET(self, httpbin): - auth = HTTPDigestAuth('user', 'wrongpass') url = httpbin('digest-auth', 'auth', 'user', 'pass') - r = requests.get(url, auth=auth) assert r.status_code == 401 - r = requests.get(url) assert r.status_code == 401 - s = requests.session() s.auth = auth r = s.get(url) assert r.status_code == 401 def test_DIGESTAUTH_QUOTES_QOP_VALUE(self, httpbin): - auth = HTTPDigestAuth('user', 'pass') url = httpbin('digest-auth', 'auth', 'user', 'pass') - r = requests.get(url, auth=auth) assert '"auth"' in r.request.headers['Authorization'] def test_POSTBIN_GET_POST_FILES(self, httpbin): - url = httpbin('post') requests.post(url).raise_for_status() - post1 = requests.post(url, data={'some': 'data'}) assert post1.status_code == 200 - with open('Pipfile') as f: post2 = requests.post(url, files={'some': f}) assert post2.status_code == 200 - post4 = requests.post(url, data='[{"some": "json"}]') assert post4.status_code == 200 - with pytest.raises(ValueError): requests.post(url, files=['bad file data']) def test_POSTBIN_SEEKED_OBJECT_WITH_NO_ITER(self, httpbin): class TestStream(object): + def __init__(self, data): self.data = data.encode() self.length = len(self.data) @@ -771,7 +745,7 @@ class TestRequests: def read(self, size=None): if size: - ret = self.data[self.index:self.index + size] + ret = self.data[self.index: self.index + size] self.index += size else: ret = self.data[self.index:] @@ -793,7 +767,6 @@ class TestRequests: post1 = requests.post(httpbin('post'), data=test) assert post1.status_code == 200 assert post1.json()['data'] == 'test' - test = TestStream('test') test.seek(2) post2 = requests.post(httpbin('post'), data=test) @@ -801,25 +774,22 @@ class TestRequests: assert post2.json()['data'] == 'st' def test_POSTBIN_GET_POST_FILES_WITH_DATA(self, httpbin): - url = httpbin('post') requests.post(url).raise_for_status() - post1 = requests.post(url, data={'some': 'data'}) assert post1.status_code == 200 - with open('Pipfile') as f: post2 = requests.post(url, data={'some': 'data'}, files={'some': f}) assert post2.status_code == 200 - post4 = requests.post(url, data='[{"some": "json"}]') assert post4.status_code == 200 - with pytest.raises(ValueError): requests.post(url, files=['bad file data']) def test_post_with_custom_mapping(self, httpbin): + class CustomMapping(collections.MutableMapping): + def __init__(self, *args, **kwargs): self.data = dict(*args, **kwargs) @@ -846,8 +816,14 @@ class TestRequests: def test_conflicting_post_params(self, httpbin): url = httpbin('post') with open('Pipfile') as f: - pytest.raises(ValueError, "requests.post(url, data='[{\"some\": \"data\"}]', files={'some': f})") - pytest.raises(ValueError, "requests.post(url, data=u('[{\"some\": \"data\"}]'), files={'some': f})") + pytest.raises( + ValueError, + "requests.post(url, data='[{\"some\": \"data\"}]', files={'some': f})", + ) + pytest.raises( + ValueError, + "requests.post(url, data=u('[{\"some\": \"data\"}]'), files={'some': f})", + ) def test_request_ok_set(self, httpbin): r = requests.get(httpbin('status', '404')) @@ -857,7 +833,6 @@ class TestRequests: r = requests.get(httpbin('status', '404')) with pytest.raises(requests.exceptions.HTTPError): r.raise_for_status() - r = requests.get(httpbin('status', '500')) assert not r.ok @@ -870,13 +845,15 @@ class TestRequests: r.content.decode('ascii') @pytest.mark.parametrize( - 'url, params', ( + 'url, params', + ( ('/get', {'foo': 'føø'}), ('/get', {'føø': 'føø'}), ('/get', {'føø': 'føø'}), ('/get', {'foo': 'foo'}), ('ø', {'foo': 'foo'}), - )) + ), + ) def test_unicode_get(self, httpbin, url, params): requests.get(httpbin(url), params=params) @@ -884,7 +861,8 @@ class TestRequests: requests.put( httpbin('put'), headers={str('Content-Type'): 'application/octet-stream'}, - data='\xff') # compat.str is unicode. + data='\xff', + ) # compat.str is unicode. def test_pyopenssl_redirect(self, httpbin_secure, httpbin_ca_bundle): requests.get(httpbin_secure('status', '301'), verify=httpbin_ca_bundle) @@ -893,17 +871,28 @@ class TestRequests: INVALID_PATH = '/garbage' with pytest.raises(IOError) as e: requests.get(httpbin_secure(), verify=INVALID_PATH) - assert str(e.value) == 'Could not find a suitable TLS CA certificate bundle, invalid path: {0}'.format(INVALID_PATH) + assert str( + e.value + ) == 'Could not find a suitable TLS CA certificate bundle, invalid path: {0}'.format( + INVALID_PATH + ) def test_invalid_ssl_certificate_files(self, httpbin_secure): INVALID_PATH = '/garbage' with pytest.raises(IOError) as e: requests.get(httpbin_secure(), cert=INVALID_PATH) - assert str(e.value) == 'Could not find the TLS certificate file, invalid path: {0}'.format(INVALID_PATH) - + assert str( + e.value + ) == 'Could not find the TLS certificate file, invalid path: {0}'.format( + INVALID_PATH + ) with pytest.raises(IOError) as e: requests.get(httpbin_secure(), cert=('.', INVALID_PATH)) - assert str(e.value) == 'Could not find the TLS key file, invalid path: {0}'.format(INVALID_PATH) + assert str( + e.value + ) == 'Could not find the TLS key file, invalid path: {0}'.format( + INVALID_PATH + ) def test_http_with_certificate(self, httpbin): r = requests.get(httpbin(), cert='.') @@ -912,22 +901,20 @@ class TestRequests: def test_https_warnings(self, httpbin_secure, httpbin_ca_bundle): """warnings are emitted with requests.get""" if HAS_MODERN_SSL or HAS_PYOPENSSL: - warnings_expected = ('SubjectAltNameWarning', ) + warnings_expected = ('SubjectAltNameWarning',) else: - warnings_expected = ('SNIMissingWarning', - 'InsecurePlatformWarning', - 'SubjectAltNameWarning', ) - + warnings_expected = ( + 'SNIMissingWarning', 'InsecurePlatformWarning', 'SubjectAltNameWarning' + ) with pytest.warns(None) as warning_records: warnings.simplefilter('always') - requests.get(httpbin_secure('status', '200'), - verify=httpbin_ca_bundle) - - warning_records = [item for item in warning_records - if item.category.__name__ != 'ResourceWarning'] - - warnings_category = tuple( - item.category.__name__ for item in warning_records) + requests.get(httpbin_secure('status', '200'), verify=httpbin_ca_bundle) + warning_records = [ + item + for item in warning_records + if item.category.__name__ != 'ResourceWarning' + ] + warnings_category = tuple(item.category.__name__ for item in warning_records) assert warnings_category == warnings_expected def test_certificate_failure(self, httpbin_secure): @@ -940,45 +927,51 @@ class TestRequests: requests.get(httpbin_secure('status', '200')) def test_urlencoded_get_query_multivalued_param(self, httpbin): - r = requests.get(httpbin('get'), params={'test': ['foo', 'baz']}) assert r.status_code == 200 assert r.url == httpbin('get?test=foo&test=baz') def test_different_encodings_dont_break_post(self, httpbin): - r = requests.post(httpbin('post'), + r = requests.post( + httpbin('post'), data={'stuff': json.dumps({'a': 123})}, params={'blah': 'asdf1234'}, - files={'file': ('test_requests.py', open(__file__, 'rb'))}) + files={'file': ('test_requests.py', open(__file__, 'rb'))}, + ) assert r.status_code == 200 - @pytest.mark.parametrize('data', + @pytest.mark.parametrize( + 'data', ( {'stuff': u('ëlïxr')}, {'stuff': u('ëlïxr').encode('utf-8')}, {'stuff': 'elixr'}, {'stuff': 'elixr'.encode('utf-8')}, - )) + ), + ) def test_unicode_multipart_post(self, httpbin, data): - r = requests.post(httpbin('post'), + r = requests.post( + httpbin('post'), data=data, - files={'file': ('test_requests.py', open(__file__, 'rb'))}) + files={'file': ('test_requests.py', open(__file__, 'rb'))}, + ) assert r.status_code == 200 def test_unicode_multipart_post_fieldnames(self, httpbin): filename = os.path.splitext(__file__)[0] + '.py' r = requests.Request( - method='POST', url=httpbin('post'), + method='POST', + url=httpbin('post'), data={'stuff'.encode('utf-8'): 'elixr'}, - files={'file': ('test_requests.py', open(filename, 'rb'))}) + files={'file': ('test_requests.py', open(filename, 'rb'))}, + ) prep = r.prepare() assert b'name="stuff"' in prep.body assert b'name="b\'stuff\'"' not in prep.body def test_unicode_method_name(self, httpbin): files = {'file': open(__file__, 'rb')} - r = requests.request( - method=u('POST'), url=httpbin('post'), files=files) + r = requests.request(method=u('POST'), url=httpbin('post'), files=files) assert r.status_code == 200 def test_unicode_method_name_with_request_object(self, httpbin): @@ -988,14 +981,12 @@ class TestRequests: prep = s.prepare_request(req) assert isinstance(prep.method, builtin_str) assert prep.method == 'POST' - resp = s.send(prep) assert resp.status_code == 200 def test_non_prepared_request_error(self): s = requests.Session() req = requests.Request(u('POST'), '/') - with pytest.raises(ValueError) as e: s.send(req) assert str(e.value) == 'You can only send PreparedRequests.' @@ -1006,12 +997,16 @@ class TestRequests: data={'stuff': json.dumps({'a': 123})}, files={ 'file1': ('test_requests.py', open(__file__, 'rb')), - 'file2': ('test_requests', open(__file__, 'rb'), - 'text/py-content-type')}) + 'file2': ( + 'test_requests', open(__file__, 'rb'), 'text/py-content-type' + ), + }, + ) assert r.status_code == 200 assert b"text/py-content-type" in r.request.body def test_hook_receives_request_arguments(self, httpbin): + def hook(resp, **kwargs): assert resp is not None assert kwargs != {} @@ -1041,36 +1036,33 @@ class TestRequests: assert prep.hooks['response'] == [hook1] def test_prepared_request_hook(self, httpbin): + def hook(resp, **kwargs): resp.hook_working = True return resp req = requests.Request('GET', httpbin(), hooks={'response': hook}) prep = req.prepare() - s = requests.Session() s.proxies = getproxies() resp = s.send(prep) - assert hasattr(resp, 'hook_working') def test_prepared_from_session(self, httpbin): + class DummyAuth(requests.auth.AuthBase): + def __call__(self, r): r.headers['Dummy-Auth-Test'] = 'dummy-auth-test-ok' return r req = requests.Request('GET', httpbin('headers')) assert not req.auth - s = requests.Session() s.auth = DummyAuth() - prep = s.prepare_request(req) resp = s.send(prep) - - assert resp.json()['headers'][ - 'Dummy-Auth-Test'] == 'dummy-auth-test-ok' + assert resp.json()['headers']['Dummy-Auth-Test'] == 'dummy-auth-test-ok' def test_prepare_request_with_bytestring_url(self): req = requests.Request('GET', b'https://httpbin.org/') @@ -1084,7 +1076,7 @@ class TestRequests: 'GET', httpbin('cookies/set?cookie=value'), allow_redirects=False, - headers={'Host': b'httpbin.org'} + headers={'Host': b'httpbin.org'}, ) assert resp.cookies.get('cookie') == 'value' @@ -1098,17 +1090,19 @@ class TestRequests: 'date': 'Sat, 26 Jan 2013 16:47:56 GMT', 'etag': '"6ff6a73c0e446c1f61614769e3ceb778"', 'last-modified': 'Sat, 26 Jan 2013 16:22:39 GMT', - 'link': ('; rel="next", ; ' - ' rel="last"'), + 'link': ( + '; rel="next", ; ' + ' rel="last"' + ), 'server': 'GitHub.com', 'status': '200 OK', 'vary': 'Accept', 'x-content-type-options': 'nosniff', 'x-github-media-type': 'github.beta', 'x-ratelimit-limit': '60', - 'x-ratelimit-remaining': '57' + 'x-ratelimit-remaining': '57', } assert r.links['next']['rel'] == 'next' @@ -1118,13 +1112,10 @@ class TestRequests: secure = True domain = 'test.com' rest = {'HttpOnly': True} - jar = requests.cookies.RequestsCookieJar() jar.set(key, value, secure=secure, domain=domain, rest=rest) - assert len(jar) == 1 assert 'some_cookie' in jar - cookie = list(jar)[0] assert cookie.secure == secure assert cookie.domain == domain @@ -1133,18 +1124,14 @@ class TestRequests: def test_cookie_as_dict_keeps_len(self): key = 'some_cookie' value = 'some_value' - key1 = 'some_cookie1' value1 = 'some_value1' - jar = requests.cookies.RequestsCookieJar() jar.set(key, value) jar.set(key1, value1) - d1 = dict(jar) d2 = dict(jar.iteritems()) d3 = dict(jar.items()) - assert len(jar) == 2 assert len(d1) == 2 assert len(d2) == 2 @@ -1153,18 +1140,14 @@ class TestRequests: def test_cookie_as_dict_keeps_items(self): key = 'some_cookie' value = 'some_value' - key1 = 'some_cookie1' value1 = 'some_value1' - jar = requests.cookies.RequestsCookieJar() jar.set(key, value) jar.set(key1, value1) - d1 = dict(jar) d2 = dict(jar.iteritems()) d3 = dict(jar.items()) - assert d1['some_cookie'] == 'some_value' assert d2['some_cookie'] == 'some_value' assert d3['some_cookie1'] == 'some_value1' @@ -1172,14 +1155,11 @@ class TestRequests: def test_cookie_as_dict_keys(self): key = 'some_cookie' value = 'some_value' - key1 = 'some_cookie1' value1 = 'some_value1' - jar = requests.cookies.RequestsCookieJar() jar.set(key, value) jar.set(key1, value1) - keys = jar.keys() assert keys == list(keys) # make sure one can use keys multiple times @@ -1188,14 +1168,11 @@ class TestRequests: def test_cookie_as_dict_values(self): key = 'some_cookie' value = 'some_value' - key1 = 'some_cookie1' value1 = 'some_value1' - jar = requests.cookies.RequestsCookieJar() jar.set(key, value) jar.set(key1, value1) - values = jar.values() assert values == list(values) # make sure one can use values multiple times @@ -1204,14 +1181,11 @@ class TestRequests: def test_cookie_as_dict_items(self): key = 'some_cookie' value = 'some_value' - key1 = 'some_cookie1' value1 = 'some_value1' - jar = requests.cookies.RequestsCookieJar() jar.set(key, value) jar.set(key1, value1) - items = jar.items() assert items == list(items) # make sure one can use items multiple times @@ -1222,18 +1196,15 @@ class TestRequests: value = 'some_value' domain1 = 'test1.com' domain2 = 'test2.com' - jar = requests.cookies.RequestsCookieJar() jar.set(key, value, domain=domain1) jar.set(key, value, domain=domain2) assert key in jar items = jar.items() assert len(items) == 2 - # Verify that CookieConflictError is raised if domain is not specified with pytest.raises(requests.cookies.CookieConflictError): jar.get(key) - # Verify that CookieConflictError is not raised if domain is specified cookie = jar.get(key, domain=domain1) assert cookie == value @@ -1242,7 +1213,6 @@ class TestRequests: key = 'some_cookie' value = 'some_value' path = 'some_path' - jar = requests.cookies.RequestsCookieJar() jar.set(key, value, path=path) jar.set(key, value) @@ -1252,7 +1222,9 @@ class TestRequests: def test_time_elapsed_blank(self, httpbin): r = requests.get(httpbin('get')) td = r.elapsed - total_seconds = ((td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6) + total_seconds = ( + (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / 10 ** 6 + ) assert total_seconds > 0.0 def test_empty_response_has_content_none(self): @@ -1266,6 +1238,7 @@ class TestRequests: def read_mock(amt, decode_content=None): return read_(amt) + setattr(io, 'read', read_mock) r.raw = io assert next(iter(r)) @@ -1279,29 +1252,23 @@ class TestRequests: r._content_consumed = True r._content = b'the content' r.encoding = 'ascii' - chunks = r.iter_content(decode_unicode=True) assert all(isinstance(chunk, str) for chunk in chunks) - # also for streaming r = requests.Response() r.raw = io.BytesIO(b'the content') r.encoding = 'ascii' - chunks = r.iter_content(decode_unicode=True) assert all(isinstance(chunk, str) for chunk in chunks) @pytest.mark.parametrize( - 'encoding, exception', ( - (None, TypeError), - ('invalid encoding', LookupError), - )) + 'encoding, exception', ((None, TypeError), ('invalid encoding', LookupError)) + ) def test_decode_unicode_encoding(self, encoding, exception): # raise an exception if encoding isn't set r = requests.Response() r.raw = io.BytesIO(b'the content') r.encoding = encoding - with pytest.raises(exception): chunks = r.iter_content(decode_unicode=True) @@ -1334,12 +1301,10 @@ class TestRequests: r.raw = io.BytesIO(b'the content') chunks = r.iter_content(1) assert all(len(chunk) == 1 for chunk in chunks) - r = requests.Response() r.raw = io.BytesIO(b'the content') chunks = r.iter_content(None) assert list(chunks) == [b'the content'] - r = requests.Response() r.raw = io.BytesIO(b'the content') with pytest.raises(TypeError): @@ -1347,17 +1312,14 @@ class TestRequests: def test_request_and_response_are_pickleable(self, httpbin): r = requests.get(httpbin('get')) - # verify we can pickle the original request assert pickle.loads(pickle.dumps(r.request)) - # verify we can pickle the response and that we have access to # the original request. pr = pickle.loads(pickle.dumps(r)) assert r.request.url == pr.request.url assert r.request.headers == pr.request.headers - def test_response_lines(self): """ iter_lines should be able to handle data dribbling in which delimiters @@ -1383,59 +1345,46 @@ class TestRequests: def mock_iter_content(*args, **kwargs): if kwargs.get("decode_unicode"): return (e.decode('utf-8') for e in mock_chunks) + return (e for e in mock_chunks) r = requests.Response() r._content_consumed = True r.iter_content = mock_iter_content - # decode_unicode=None, output raw bytes assert list(r.iter_lines(delimiter=b'\r\n')) == mock_data.split(b'\r\n') - # decode_unicode=True, output unicode strings - assert list(r.iter_lines(decode_unicode=True, delimiter=u'\r\n')) == unicode_mock_data.split(u'\r\n') - + assert list( + r.iter_lines(decode_unicode=True, delimiter=u'\r\n') + ) == unicode_mock_data.split( + u'\r\n' + ) # When delimiter is None, we should yield the same result as splitlines() # which supports the universal newline. # '\r', '\n', and '\r\n' are all treated as one line break. - # decode_unicode=None, output raw bytes result = list(r.iter_lines()) assert result == mock_data.splitlines() - # decode_unicode=True, output unicode strings result = list(r.iter_lines(decode_unicode=True)) assert result == unicode_mock_data.splitlines() - # If we change all the line breaks to `\r`, we should be okay. # decode_unicode=None, output raw bytes mock_chunks = [chunk.replace(b'\n', b'\r') for chunk in mock_chunks] mock_data = b''.join(mock_chunks) assert list(r.iter_lines()) == mock_data.splitlines() - # decode_unicode=True, output unicode strings unicode_mock_data = mock_data.decode('utf-8') assert list(r.iter_lines(decode_unicode=True)) == unicode_mock_data.splitlines() - @pytest.mark.parametrize( - 'content, expected_no_delimiter, expected_delimiter', ( - ([b''], [], []), - ([b'line\n'], [u'line'], [u'line\n']), - ([b'line', b'\n'], [u'line'], [u'line\n']), - ([b'line\r\n'], [u'line'], [u'line', u'']), - # Empty chunk in the end of stream, same behavior as the previous - ([b'line\r\n', b''], [u'line'], [u'line', u'']), - ([b'line', b'\r\n'], [u'line'], [u'line', u'']), - ([b'a\r', b'\nb\r'], [u'a', u'b'], [u'a', u'b\r']), - ([b'a\r', b'\n', b'\nb'], [u'a', u'', u'b'], [u'a', u'\nb']), - ([b'a\n', b'\nb'], [u'a', u'', u'b'], [u'a\n\nb']), - ([b'a\r\n', b'\rb\n'], [u'a', u'', u'b'], [u'a', u'\rb\n']), - ([b'a\nb', b'c'], [u'a', u'bc'], [u'a\nbc']), - ([b'a\n', b'\rb', b'\r\nc'], [u'a', u'', u'b', u'c'], [u'a\n\rb', u'c']), - ([b'a\r\nb', b'', b'c'], [u'a', u'bc'], [u'a', u'bc']) # Empty chunk with pending data - )) - def test_response_lines_parametrized(self, content, expected_no_delimiter, expected_delimiter): + 'content, expected_no_delimiter, expected_delimiter', + (([b''], [], []), ([b'line\n'], [u'line'], [u'line\n']), ([b'line', b'\n'], [u'line'], [u'line\n']), ([b'line\r\n'], [u'line'], [u'line', u'']), ([b'line\r\n', b''], [u'line'], [u'line', u'']), ([b'line', b'\r\n'], [u'line'], [u'line', u'']), ([b'a\r', b'\nb\r'], [u'a', u'b'], [u'a', u'b\r']), ([b'a\r', b'\n', b'\nb'], [u'a', u'', u'b'], [u'a', u'\nb']), ([b'a\n', b'\nb'], [u'a', u'', u'b'], [u'a\n\nb']), ([b'a\r\n', b'\rb\n'], [u'a', u'', u'b'], [u'a', u'\rb\n']), ([b'a\nb', b'c'], [u'a', u'bc'], [u'a\nbc']), ([b'a\n', b'\rb', b'\r\nc'], [u'a', u'', u'b', u'c'], [u'a\n\rb', u'c']), ([b'a\r\nb', b'', b'c'], [u'a', u'bc'], [u'a', u'bc'])), + # Empty chunk in the end of stream, same behavior as the previous # Empty chunk with pending data + ) + def test_response_lines_parametrized( + self, content, expected_no_delimiter, expected_delimiter + ): """ Test a lot of potential chunk splits to ensure consistency of iter_lines(delimiter=x), as well as the legacy behavior of @@ -1443,32 +1392,36 @@ class TestRequests: https://github.com/kennethreitz/requests/pull/2431#issuecomment-72333964 """ mock_chunks = content + def mock_iter_content(*args, **kwargs): if kwargs.get("decode_unicode"): return (e.decode('utf-8') for e in mock_chunks) + return (e for e in mock_chunks) r = requests.Response() r._content_consumed = True r.iter_content = mock_iter_content - # decode_unicode=True, output unicode strings assert list(r.iter_lines(decode_unicode=True)) == expected_no_delimiter - assert list(r.iter_lines(decode_unicode=True, delimiter='\r\n')) == expected_delimiter - + assert list( + r.iter_lines(decode_unicode=True, delimiter='\r\n') + ) == expected_delimiter # decode_unicode=None, output raw bytes - assert list(r.iter_lines()) == [line.encode('utf-8') for line in expected_no_delimiter] - assert list(r.iter_lines(delimiter=b'\r\n')) == [line.encode('utf-8') for line in expected_delimiter] + assert list(r.iter_lines()) == [ + line.encode('utf-8') for line in expected_no_delimiter + ] + assert list(r.iter_lines(delimiter=b'\r\n')) == [ + line.encode('utf-8') for line in expected_delimiter + ] def test_prepared_request_is_pickleable(self, httpbin): p = requests.Request('GET', httpbin('get')).prepare() - # Verify PreparedRequest can be pickled and unpickled r = pickle.loads(pickle.dumps(p)) assert r.url == p.url assert r.headers == p.headers assert r.body == p.body - # Verify unpickled PreparedRequest sends properly s = requests.Session() resp = s.send(r) @@ -1478,13 +1431,11 @@ class TestRequests: files = {'file': open(__file__, 'rb')} r = requests.Request('POST', httpbin('post'), files=files) p = r.prepare() - # Verify PreparedRequest can be pickled and unpickled r = pickle.loads(pickle.dumps(p)) assert r.url == p.url assert r.headers == p.headers assert r.body == p.body - # Verify unpickled PreparedRequest sends properly s = requests.Session() resp = s.send(r) @@ -1493,14 +1444,12 @@ class TestRequests: def test_prepared_request_with_hook_is_pickleable(self, httpbin): r = requests.Request('GET', httpbin('get'), hooks=default_hooks()) p = r.prepare() - # Verify PreparedRequest can be pickled r = pickle.loads(pickle.dumps(p)) assert r.url == p.url assert r.headers == p.headers assert r.body == p.body assert r.hooks == p.hooks - # Verify unpickled PreparedRequest sends properly s = requests.Session() resp = s.send(r) @@ -1524,10 +1473,8 @@ class TestRequests: def test_session_pickling(self, httpbin): r = requests.Request('GET', httpbin('get')) s = requests.Session() - s = pickle.loads(pickle.dumps(s)) s.proxies = getproxies() - r = s.send(r.prepare()) assert r.status_code == 200 @@ -1613,7 +1560,6 @@ class TestRequests: headers = {u('unicode'): 'blah', 'byte'.encode('ascii'): 'blah'} r = requests.Request('GET', httpbin('get'), headers=headers) p = r.prepare() - # This is testing that they are builtin strings. A bit weird, but there # we go. assert 'unicode' in p.headers.keys() @@ -1621,10 +1567,9 @@ class TestRequests: def test_header_validation(self, httpbin): """Ensure prepare_headers regex isn't flagging valid header contents.""" - headers_ok = {'foo': 'bar baz qux', - 'bar': u'fbbq'.encode('utf8'), - 'baz': '', - 'qux': '1'} + headers_ok = { + 'foo': 'bar baz qux', 'bar': u'fbbq'.encode('utf8'), 'baz': '', 'qux': '1' + } r = requests.get(httpbin('get'), headers=headers_ok) assert r.request.headers['foo'] == headers_ok['foo'] @@ -1635,7 +1580,6 @@ class TestRequests: headers_int = {'foo': 3} headers_dict = {'bar': {'foo': 'bar'}} headers_list = {'baz': ['foo', 'bar']} - # Test for int with pytest.raises(InvalidHeader) as excinfo: r = requests.get(httpbin('get'), headers=headers_int) @@ -1656,7 +1600,6 @@ class TestRequests: headers_ret = {'foo': 'bar\r\nbaz: qux'} headers_lf = {'foo': 'bar\nbaz: qux'} headers_cr = {'foo': 'bar\rbaz: qux'} - # Test for newline with pytest.raises(InvalidHeader): r = requests.get(httpbin('get'), headers=headers_ret) @@ -1673,7 +1616,6 @@ class TestRequests: """ headers_space = {'foo': ' bar'} headers_tab = {'foo': ' bar'} - # Test for whitespace with pytest.raises(InvalidHeader): r = requests.get(httpbin('get'), headers=headers_space) @@ -1694,7 +1636,6 @@ class TestRequests: f.name = 2 r = requests.Request('POST', httpbin('post'), files={'f': f}) p = r.prepare() - assert 'multipart/form-data' in p.headers['Content-Type'] def test_autoset_header_values_are_native(self, httpbin): @@ -1702,7 +1643,6 @@ class TestRequests: length = '16' req = requests.Request('POST', httpbin('post'), data=data) p = req.prepare() - assert p.headers['Content-Length'] == length def test_nonhttp_schemes_dont_check_URLs(self): @@ -1730,7 +1670,6 @@ class TestRequests: r = requests.get(httpbin('redirect/1'), auth=('user', 'pass')) h1 = r.history[0].request.headers['Authorization'] h2 = r.request.headers['Authorization'] - assert h1 == h2 def test_manual_redirect_with_partial_body_read(self, httpbin): @@ -1739,13 +1678,11 @@ class TestRequests: r1 = s.send(req, allow_redirects=False, stream=True) assert r1.is_redirect rg = s.resolve_redirects(r1, req, stream=True) - # read only the first eight bytes of the response body, # then follow the redirect r1.iter_content(8) r2 = next(rg) assert r2.is_redirect - # read all of the response via iter_content, # then follow the redirect for _ in r2.iter_content(): @@ -1765,10 +1702,8 @@ class TestRequests: prep = requests.Request('GET', 'http://example.com', data=data).prepare() assert prep._body_position == 0 assert prep.body.read() == b'the data' - # the data has all been read assert prep.body.read() == b'' - # rewind it back requests.utils.rewind_body(prep) assert prep.body.read() == b'the data' @@ -1780,16 +1715,16 @@ class TestRequests: prep = requests.Request('GET', 'http://example.com', data=data).prepare() assert prep._body_position == 4 assert prep.body.read() == b'data' - # the data has all been read assert prep.body.read() == b'' - # rewind it back requests.utils.rewind_body(prep) assert prep.body.read() == b'data' def test_rewind_body_no_seek(self): + class BadFileObj: + def __init__(self, data): self.data = data @@ -1803,14 +1738,14 @@ class TestRequests: s = requests.Session() prep = requests.Request('GET', 'http://example.com', data=data).prepare() assert prep._body_position == 0 - with pytest.raises(UnrewindableBodyError) as e: requests.utils.rewind_body(prep) - assert 'Unable to rewind request body' in str(e) def test_rewind_body_failed_seek(self): + class BadFileObj: + def __init__(self, data): self.data = data @@ -1827,14 +1762,14 @@ class TestRequests: s = requests.Session() prep = requests.Request('GET', 'http://example.com', data=data).prepare() assert prep._body_position == 0 - with pytest.raises(UnrewindableBodyError) as e: requests.utils.rewind_body(prep) - assert 'error occurred when rewinding request body' in str(e) def test_rewind_body_failed_tell(self): + class BadFileObj: + def __init__(self, data): self.data = data @@ -1848,10 +1783,8 @@ class TestRequests: s = requests.Session() prep = requests.Request('GET', 'http://example.com', data=data).prepare() assert prep._body_position is not None - with pytest.raises(UnrewindableBodyError) as e: requests.utils.rewind_body(prep) - assert 'Unable to rewind request body' in str(e) def _patch_adapter_gzipped_redirect(self, session, url): @@ -1875,10 +1808,16 @@ class TestRequests: s.get(url) @pytest.mark.parametrize( - 'username, password, auth_str', ( + 'username, password, auth_str', + ( ('test', 'test', 'Basic dGVzdDp0ZXN0'), - (u'имя'.encode('utf-8'), u'пароль'.encode('utf-8'), 'Basic 0LjQvNGPOtC/0LDRgNC+0LvRjA=='), - )) + ( + u'имя'.encode('utf-8'), + u'пароль'.encode('utf-8'), + 'Basic 0LjQvNGPOtC/0LDRgNC+0LvRjA==', + ), + ), + ) def test_basic_auth_str_is_always_native(self, username, password, auth_str): s = _basic_auth_str(username, password) assert isinstance(s, builtin_str) @@ -1893,18 +1832,18 @@ class TestRequests: i += 1 def test_json_param_post_content_type_works(self, httpbin): - r = requests.post( - httpbin('post'), - json={'life': 42} - ) + r = requests.post(httpbin('post'), json={'life': 42}) assert r.status_code == 200 assert 'application/json' in r.request.headers['Content-Type'] assert {'life': 42} == r.json()['json'] def test_json_param_post_should_not_override_data_param(self, httpbin): - r = requests.Request(method='POST', url=httpbin('post'), - data={'stuff': 'elixr'}, - json={'music': 'flute'}) + r = requests.Request( + method='POST', + url=httpbin('post'), + data={'stuff': 'elixr'}, + json={'music': 'flute'}, + ) prep = r.prepare() assert 'stuff=elixr' == prep.body @@ -1920,15 +1859,12 @@ class TestRequests: def test_response_context_manager(self, httpbin): with requests.get(httpbin('stream/4'), stream=True) as response: assert isinstance(response, requests.Response) - assert response.raw.closed def test_unconsumed_session_response_closes_connection(self, httpbin): s = requests.session() - with contextlib.closing(s.get(httpbin('stream/4'), stream=True)) as response: pass - assert response._content_consumed is False assert response.raw.closed @@ -1937,7 +1873,6 @@ class TestRequests: """Response.iter_lines() is not reentrant safe""" r = requests.get(httpbin('stream/4'), stream=True) assert r.status_code == 200 - next(r.iter_lines()) assert len(list(r.iter_lines())) == 3 @@ -1947,34 +1882,27 @@ class TestRequests: s = requests.Session() a = SendRecordingAdapter() s.mount('http://', a) - # Both of these arguments are safe fallbacks that we can easily # detect, but which will allow the request to succeed. s.verify = False s.proxies = {'http': None} - old_proxy = os.environ.get('HTTP_PROXY') old_bundle = os.environ.get('REQUESTS_CA_BUNDLE') - try: os.environ['HTTP_PROXY'] = '10.10.10.10:3128' os.environ['REQUESTS_CA_BUNDLE'] = '/path/to/nowhere' - s.get(httpbin('get'), timeout=5) finally: if old_proxy is not None: os.environ['HTTP_PROXY'] = old_proxy else: del os.environ['HTTP_PROXY'] - if old_bundle is not None: os.environ['REQUESTS_CA_BUNDLE'] = old_bundle else: del os.environ['REQUESTS_CA_BUNDLE'] - call = a.send_calls[0] assert call[1]['verify'] == False - proxies = call[1]['proxies'] with pytest.raises(KeyError): proxies['http'] @@ -1985,20 +1913,17 @@ class TestRequests: session = requests.Session() monkeypatch.delenv('CURL_CA_BUNDLE', raising=False) monkeypatch.delenv('REQUESTS_CA_BUNDLE', raising=False) - assert session.trust_env is True assert session.verify is True assert 'REQUESTS_CA_BUNDLE' not in os.environ assert 'CURL_CA_BUNDLE' not in os.environ merged_settings = session.merge_environment_settings( - 'http://example.com', {}, False, True, None) + 'http://example.com', {}, False, True, None + ) assert merged_settings['verify'] is True def test_session_close_proxy_clear(self, mocker): - proxies = { - 'one': mocker.Mock(), - 'two': mocker.Mock(), - } + proxies = {'one': mocker.Mock(), 'two': mocker.Mock()} session = requests.Session() mocker.patch.dict(session.adapters['http://'].proxy_manager, proxies) session.close() @@ -2021,7 +1946,6 @@ class TestRequests: r.status_code = 0 r._content = False r._content_consumed = False - assert r.content is None with pytest.raises(ValueError): r.json() @@ -2098,7 +2022,9 @@ class TestRequests: assert 'Transfer-Encoding' in prepared_request.headers assert 'Content-Length' not in prepared_request.headers - def test_chunked_upload_with_manually_set_content_length_header_raises_error(self, httpbin): + def test_chunked_upload_with_manually_set_content_length_header_raises_error( + self, httpbin + ): """Ensure that if a user manually sets a content length header, when the data is chunked, that an InvalidHeader error is raised. """ @@ -2107,7 +2033,9 @@ class TestRequests: with pytest.raises(InvalidHeader): r = requests.post(url, data=data, headers={'Content-Length': 'foo'}) - def test_content_length_with_manually_set_transfer_encoding_raises_error(self, httpbin): + def test_content_length_with_manually_set_transfer_encoding_raises_error( + self, httpbin + ): """Ensure that if a user manually sets a Transfer-Encoding header when data is not chunked that an InvalidHeader error is raised. """ @@ -2124,18 +2052,19 @@ class TestRequests: pytest.fail('InvalidHeader error raised unexpectedly.') @pytest.mark.parametrize( - 'body, expected', ( + 'body, expected', + ( (None, ('Content-Length', '0')), ('test_data', ('Content-Length', '9')), (io.BytesIO(b'test_data'), ('Content-Length', '9')), - (StringIO.StringIO(''), ('Transfer-Encoding', 'chunked')) - )) + (StringIO.StringIO(''), ('Transfer-Encoding', 'chunked')), + ), + ) def test_prepare_content_length(self, httpbin, body, expected): """Test prepare_content_length creates expected header.""" prep = requests.PreparedRequest() prep.headers = {} prep.method = 'POST' - # Ensure Content-Length is set appropriately. key, value = expected prep.prepare_content_length(body) @@ -2147,7 +2076,6 @@ class TestRequests: prep = requests.PreparedRequest() prep.headers = {} prep.method = 'POST' - with pytest.raises(InvalidBodyError) as e: # Send object that isn't iterable and has no accessible content. prep.prepare_content_length(object()) @@ -2170,20 +2098,20 @@ class TestRequests: url_redirect_malformed = httpbin('response-headers?%s' % querystring_malformed) querystring_redirect = urlencode({'url': url_redirect_malformed}) url_redirect = httpbin('redirect-to?%s' % querystring_redirect) - urls_test = [url_redirect, - url_redirect_malformed, - url_final, - ] + urls_test = [url_redirect, url_redirect_malformed, url_final] class CustomRedirectSession(requests.Session): + def get_redirect_target(self, resp): # default behavior if resp.is_redirect: return resp.headers['location'] + # edge case - check to see if 'location' is in headers anyways location = resp.headers.get('location') if location and (location != resp.url): return location + return None session = CustomRedirectSession() @@ -2200,11 +2128,13 @@ class TestRequests: class TestCaseInsensitiveDict: @pytest.mark.parametrize( - 'cid', ( + 'cid', + ( CaseInsensitiveDict({'Foo': 'foo', 'BAr': 'bar'}), CaseInsensitiveDict([('Foo', 'foo'), ('BAr', 'bar')]), CaseInsensitiveDict(FOO='foo', BAr='bar'), - )) + ), + ) def test_init(self, cid): assert len(cid) == 2 assert 'foo' in cid @@ -2298,29 +2228,26 @@ class TestCaseInsensitiveDict: assert cid.setdefault('notspam', 'notblueval') == 'notblueval' def test_lower_items(self): - cid = CaseInsensitiveDict({ - 'Accept': 'application/json', - 'user-Agent': 'requests', - }) + cid = CaseInsensitiveDict( + {'Accept': 'application/json', 'user-Agent': 'requests'} + ) keyset = frozenset(lowerkey for lowerkey, v in cid.lower_items()) lowerkeyset = frozenset(['accept', 'user-agent']) assert keyset == lowerkeyset def test_preserve_key_case(self): - cid = CaseInsensitiveDict({ - 'Accept': 'application/json', - 'user-Agent': 'requests', - }) + cid = CaseInsensitiveDict( + {'Accept': 'application/json', 'user-Agent': 'requests'} + ) keyset = frozenset(['Accept', 'user-Agent']) assert frozenset(i[0] for i in cid.items()) == keyset assert frozenset(cid.keys()) == keyset assert frozenset(cid) == keyset def test_preserve_last_key_case(self): - cid = CaseInsensitiveDict({ - 'Accept': 'application/json', - 'user-Agent': 'requests', - }) + cid = CaseInsensitiveDict( + {'Accept': 'application/json', 'user-Agent': 'requests'} + ) cid.update({'ACCEPT': 'application/json'}) cid['USER-AGENT'] = 'requests' keyset = frozenset(['ACCEPT', 'USER-AGENT']) @@ -2329,10 +2256,9 @@ class TestCaseInsensitiveDict: assert frozenset(cid) == keyset def test_copy(self): - cid = CaseInsensitiveDict({ - 'Accept': 'application/json', - 'user-Agent': 'requests', - }) + cid = CaseInsensitiveDict( + {'Accept': 'application/json', 'user-Agent': 'requests'} + ) cid_copy = cid.copy() assert cid == cid_copy cid['changed'] = True @@ -2356,17 +2282,14 @@ class TestMorselToCookieExpires: def test_expires_valid_str(self): """Test case where we convert expires from string time.""" - morsel = Morsel() morsel['expires'] = 'Thu, 01-Jan-1970 00:00:01 GMT' cookie = morsel_to_cookie(morsel) assert cookie.expires == 1 @pytest.mark.parametrize( - 'value, exception', ( - (100, TypeError), - ('woops', ValueError), - )) + 'value, exception', ((100, TypeError), ('woops', ValueError)) + ) def test_expires_invalid_int(self, value, exception): """Test case where an invalid type is passed for expires.""" morsel = Morsel() @@ -2376,7 +2299,6 @@ class TestMorselToCookieExpires: def test_expires_none(self): """Test case where expires is None.""" - morsel = Morsel() morsel['expires'] = None cookie = morsel_to_cookie(morsel) @@ -2384,12 +2306,10 @@ class TestMorselToCookieExpires: class TestMorselToCookieMaxAge: - """Tests for morsel_to_cookie when morsel contains max-age.""" def test_max_age_valid_int(self): """Test case where a valid max age in seconds is passed.""" - morsel = Morsel() morsel['max-age'] = 60 cookie = morsel_to_cookie(morsel) @@ -2397,7 +2317,6 @@ class TestMorselToCookieMaxAge: def test_max_age_invalid_str(self): """Test case where a invalid max age is passed.""" - morsel = Morsel() morsel['max-age'] = 'woops' with pytest.raises(TypeError): @@ -2413,20 +2332,15 @@ class TestTimeout: assert 'Read timed out' in e.args[0].args[0] @pytest.mark.parametrize( - 'timeout, error_text', ( - ((3, 4, 5), '(connect, read)'), - ('foo', 'must be an int, float or None'), - )) + 'timeout, error_text', + (((3, 4, 5), '(connect, read)'), ('foo', 'must be an int, float or None')), + ) def test_invalid_timeout(self, httpbin, timeout, error_text): with pytest.raises(ValueError) as e: requests.get(httpbin('get'), timeout=timeout) assert error_text in str(e) - @pytest.mark.parametrize( - 'timeout', ( - None, - Urllib3Timeout(connect=None, read=None) - )) + @pytest.mark.parametrize('timeout', (None, Urllib3Timeout(connect=None, read=None))) def test_none_timeout(self, httpbin, timeout): """Check that you can set None as a valid timeout value. @@ -2440,10 +2354,8 @@ class TestTimeout: assert r.status_code == 200 @pytest.mark.parametrize( - 'timeout', ( - (None, 0.1), - Urllib3Timeout(connect=None, read=0.1) - )) + 'timeout', ((None, 0.1), Urllib3Timeout(connect=None, read=0.1)) + ) def test_read_timeout(self, httpbin, timeout): try: requests.get(httpbin('delay/10'), timeout=timeout) @@ -2452,10 +2364,8 @@ class TestTimeout: pass @pytest.mark.parametrize( - 'timeout', ( - (0.1, None), - Urllib3Timeout(connect=0.1, read=None) - )) + 'timeout', ((0.1, None), Urllib3Timeout(connect=0.1, read=None)) + ) def test_connect_timeout(self, timeout): try: requests.get(TARPIT, timeout=timeout) @@ -2465,10 +2375,8 @@ class TestTimeout: assert isinstance(e, Timeout) @pytest.mark.parametrize( - 'timeout', ( - (0.1, 0.1), - Urllib3Timeout(connect=0.1, read=0.1) - )) + 'timeout', ((0.1, 0.1), Urllib3Timeout(connect=0.1, read=0.1)) + ) def test_total_timeout_connect(self, timeout): try: requests.get(TARPIT, timeout=timeout) @@ -2486,6 +2394,7 @@ SendCall = collections.namedtuple('SendCall', ('args', 'kwargs')) class RedirectSession(SessionRedirectMixin): + def __init__(self, order_of_redirects): self.redirects = order_of_redirects self.calls = [] @@ -2501,12 +2410,10 @@ class RedirectSession(SessionRedirectMixin): def build_response(self): request = self.calls[-1].args[0] r = requests.Response() - try: r.status_code = int(self.redirects.pop(0)) except IndexError: r.status_code = 200 - r.headers = CaseInsensitiveDict({'Location': self.location}) r.raw = self._build_raw() r.request = request @@ -2522,11 +2429,7 @@ def test_json_encodes_as_bytes(): # urllib3 expects bodies as bytes-like objects body = {"key": "value"} p = PreparedRequest() - p.prepare( - method='GET', - url='https://www.example.com/', - json=body - ) + p.prepare(method='GET', url='https://www.example.com/', json=body) assert isinstance(p.body, bytes) @@ -2551,20 +2454,20 @@ def test_requests_are_updated_each_time(httpbin): assert session.calls[-1] == send_call -@pytest.mark.parametrize("var,url,proxy", [ - ('http_proxy', 'http://example.com', 'socks5://proxy.com:9876'), - ('https_proxy', 'https://example.com', 'socks5://proxy.com:9876'), - ('all_proxy', 'http://example.com', 'socks5://proxy.com:9876'), - ('all_proxy', 'https://example.com', 'socks5://proxy.com:9876'), -]) +@pytest.mark.parametrize( + "var,url,proxy", + [ + ('http_proxy', 'http://example.com', 'socks5://proxy.com:9876'), + ('https_proxy', 'https://example.com', 'socks5://proxy.com:9876'), + ('all_proxy', 'http://example.com', 'socks5://proxy.com:9876'), + ('all_proxy', 'https://example.com', 'socks5://proxy.com:9876'), + ], +) def test_proxy_env_vars_override_default(var, url, proxy): session = requests.Session() prep = PreparedRequest() prep.prepare(method='GET', url=url) - - kwargs = { - var: proxy - } + kwargs = {var: proxy} scheme = urlparse(url).scheme with override_environ(**kwargs): proxies = session.rebuild_proxies(prep, {}) @@ -2573,46 +2476,44 @@ def test_proxy_env_vars_override_default(var, url, proxy): @pytest.mark.parametrize( - 'data', ( + 'data', + ( (('a', 'b'), ('c', 'd')), (('c', 'd'), ('a', 'b')), (('a', 'b'), ('c', 'd'), ('e', 'f')), - )) + ), +) def test_data_argument_accepts_tuples(data): """Ensure that the data argument will accept tuples of strings and properly encode them. """ p = PreparedRequest() p.prepare( - method='GET', - url='http://www.example.com', - data=data, - hooks=default_hooks() + method='GET', url='http://www.example.com', data=data, hooks=default_hooks() ) assert p.body == urlencode(data) @pytest.mark.parametrize( - 'kwargs', ( + 'kwargs', + ( None, { 'method': 'GET', 'url': 'http://www.example.com', 'data': 'foo=bar', - 'hooks': default_hooks() + 'hooks': default_hooks(), }, { 'method': 'GET', 'url': 'http://www.example.com', 'data': 'foo=bar', 'hooks': default_hooks(), - 'cookies': {'foo': 'bar'} + 'cookies': {'foo': 'bar'}, }, - { - 'method': 'GET', - 'url': u('http://www.example.com/üniçø∂é') - }, - )) + {'method': 'GET', 'url': u('http://www.example.com/üniçø∂é')}, + ), +) def test_prepared_copy(kwargs): p = PreparedRequest() if kwargs: @@ -2626,7 +2527,6 @@ def test_prepare_requires_a_request_method(): req = requests.Request() with pytest.raises(ValueError): req.prepare() - prepped = PreparedRequest() with pytest.raises(ValueError): prepped.prepare() @@ -2634,11 +2534,9 @@ def test_prepare_requires_a_request_method(): def test_urllib3_retries(httpbin): from urllib3.util import Retry - s = requests.Session() - s.mount('http://', HTTPAdapter(max_retries=Retry( - total=2, status_forcelist=[500] - ))) + s = requests.Session() + s.mount('http://', HTTPAdapter(max_retries=Retry(total=2, status_forcelist=[500]))) with pytest.raises(RetryError): s.get(httpbin('status/500')) @@ -2646,7 +2544,6 @@ def test_urllib3_retries(httpbin): def test_urllib3_pool_connection_closed(httpbin): s = requests.Session() s.mount('http://', HTTPAdapter(pool_connections=0, pool_maxsize=0)) - try: s.get(httpbin('status/200')) except ConnectionError as e: @@ -2654,45 +2551,37 @@ def test_urllib3_pool_connection_closed(httpbin): class TestPreparingURLs(object): + @pytest.mark.parametrize( 'url,expected', ( ('http://google.com', 'http://google.com/'), (u'http://ジェーピーニック.jp', u'http://xn--hckqz9bzb1cyrb.jp/'), (u'http://xn--n3h.net/', u'http://xn--n3h.net/'), - ( - u'http://ジェーピーニック.jp'.encode('utf-8'), - u'http://xn--hckqz9bzb1cyrb.jp/' - ), - ( - u'http://straße.de/straße', - u'http://xn--strae-oqa.de/stra%C3%9Fe' - ), + (u'http://ジェーピーニック.jp'.encode('utf-8'), u'http://xn--hckqz9bzb1cyrb.jp/'), + (u'http://straße.de/straße', u'http://xn--strae-oqa.de/stra%C3%9Fe'), ( u'http://straße.de/straße'.encode('utf-8'), - u'http://xn--strae-oqa.de/stra%C3%9Fe' + u'http://xn--strae-oqa.de/stra%C3%9Fe', ), ( u'http://Königsgäßchen.de/straße', - u'http://xn--knigsgchen-b4a3dun.de/stra%C3%9Fe' + u'http://xn--knigsgchen-b4a3dun.de/stra%C3%9Fe', ), ( u'http://Königsgäßchen.de/straße'.encode('utf-8'), - u'http://xn--knigsgchen-b4a3dun.de/stra%C3%9Fe' - ), - ( - b'http://xn--n3h.net/', - u'http://xn--n3h.net/' + u'http://xn--knigsgchen-b4a3dun.de/stra%C3%9Fe', ), + (b'http://xn--n3h.net/', u'http://xn--n3h.net/'), ( b'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/', - u'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/' + u'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/', ), ( u'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/', - u'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/' - ) - ) + u'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/', + ), + ), ) def test_preparing_url(self, url, expected): r = requests.Request('GET', url=url) @@ -2706,8 +2595,8 @@ class TestPreparingURLs(object): b"http://*", u"http://*.google.com", u"http://*", - u"http://☃.net/" - ) + u"http://☃.net/", + ), ) def test_preparing_bad_url(self, url): r = requests.Request('GET', url=url) @@ -2725,19 +2614,10 @@ class TestPreparingURLs(object): u"http+unix://%2Fvar%2Frun%2Fsocket/path%7E", u"http+unix://%2Fvar%2Frun%2Fsocket/path~", ), - ( - b"mailto:user@example.org", - u"mailto:user@example.org", - ), - ( - u"mailto:user@example.org", - u"mailto:user@example.org", - ), - ( - b"data:SSDimaUgUHl0aG9uIQ==", - u"data:SSDimaUgUHl0aG9uIQ==", - ) - ) + (b"mailto:user@example.org", u"mailto:user@example.org"), + (u"mailto:user@example.org", u"mailto:user@example.org"), + (b"data:SSDimaUgUHl0aG9uIQ==", u"data:SSDimaUgUHl0aG9uIQ=="), + ), ) def test_url_mutation(self, input, expected): """ @@ -2763,17 +2643,9 @@ class TestPreparingURLs(object): {"key": "value"}, u"http+unix://%2Fvar%2Frun%2Fsocket/path?key=value", ), - ( - b"mailto:user@example.org", - {"key": "value"}, - u"mailto:user@example.org", - ), - ( - u"mailto:user@example.org", - {"key": "value"}, - u"mailto:user@example.org", - ), - ) + (b"mailto:user@example.org", {"key": "value"}, u"mailto:user@example.org"), + (u"mailto:user@example.org", {"key": "value"}, u"mailto:user@example.org"), + ), ) def test_parameters_for_nonstandard_schemes(self, input, params, expected): """ @@ -2790,6 +2662,7 @@ class TestGetConnection(object): Tests for the :meth:`requests.adapters.HTTPAdapter.get_connection` that assert the connections are correctly configured. """ + @pytest.mark.parametrize( 'proxies, verify, cert, expected', ( @@ -2890,7 +2763,10 @@ class TestGetConnection(object): }, ), ( - {'http': 'http://proxy.example.com', 'https': 'http://proxy.example.com'}, + { + 'http': 'http://proxy.example.com', + 'https': 'http://proxy.example.com', + }, True, None, { @@ -2902,7 +2778,10 @@ class TestGetConnection(object): }, ), ( - {'http': 'http://proxy.example.com', 'https': 'http://proxy.example.com'}, + { + 'http': 'http://proxy.example.com', + 'https': 'http://proxy.example.com', + }, os.path.dirname(__file__), None, { @@ -2914,7 +2793,10 @@ class TestGetConnection(object): }, ), ( - {'http': 'http://proxy.example.com', 'https': 'http://proxy.example.com'}, + { + 'http': 'http://proxy.example.com', + 'https': 'http://proxy.example.com', + }, __file__, None, { @@ -2926,7 +2808,10 @@ class TestGetConnection(object): }, ), ( - {'http': 'http://proxy.example.com', 'https': 'http://proxy.example.com'}, + { + 'http': 'http://proxy.example.com', + 'https': 'http://proxy.example.com', + }, True, __file__, { @@ -2938,7 +2823,10 @@ class TestGetConnection(object): }, ), ( - {'http': 'http://proxy.example.com', 'https': 'http://proxy.example.com'}, + { + 'http': 'http://proxy.example.com', + 'https': 'http://proxy.example.com', + }, True, (__file__, __file__), { @@ -2949,13 +2837,14 @@ class TestGetConnection(object): 'key_file': __file__, }, ), - ) + ), ) def test_get_https_connection(self, proxies, verify, cert, expected): """Assert connections are configured correctly.""" adapter = requests.adapters.HTTPAdapter() connection = adapter.get_connection( - 'https://example.com', proxies=proxies, verify=verify, cert=cert) + 'https://example.com', proxies=proxies, verify=verify, cert=cert + ) actual_config = {} for key, value in connection.__dict__.items(): if key in expected: @@ -2969,7 +2858,7 @@ class TestGetConnection(object): (True, 'a/path/that/does/not/exist'), (True, (__file__, 'a/path/that/does/not/exist')), (True, ('a/path/that/does/not/exist', __file__)), - ) + ), ) def test_cert_files_missing(self, verify, cert): """ diff --git a/tests/test_structures.py b/tests/test_structures.py index e4d2459f..f1cbfb98 100644 --- a/tests/test_structures.py +++ b/tests/test_structures.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - import pytest from requests.structures import CaseInsensitiveDict, LookupDict @@ -16,7 +15,9 @@ class TestCaseInsensitiveDict: def test_list(self): assert list(self.case_insensitive_dict) == ['Accept'] - possible_keys = pytest.mark.parametrize('key', ('accept', 'ACCEPT', 'aCcEpT', 'Accept')) + possible_keys = pytest.mark.parametrize( + 'key', ('accept', 'ACCEPT', 'aCcEpT', 'Accept') + ) @possible_keys def test_getitem(self, key): @@ -28,7 +29,9 @@ class TestCaseInsensitiveDict: assert key not in self.case_insensitive_dict def test_lower_items(self): - assert list(self.case_insensitive_dict.lower_items()) == [('accept', 'application/json')] + assert list(self.case_insensitive_dict.lower_items()) == [ + ('accept', 'application/json') + ] def test_repr(self): assert repr(self.case_insensitive_dict) == "{'Accept': 'application/json'}" @@ -39,11 +42,8 @@ class TestCaseInsensitiveDict: assert copy == self.case_insensitive_dict @pytest.mark.parametrize( - 'other, result', ( - ({'AccePT': 'application/json'}, True), - ({}, False), - (None, False) - ) + 'other, result', + (({'AccePT': 'application/json'}, True), ({}, False), (None, False)), ) def test_instance_equality(self, other, result): assert (self.case_insensitive_dict == other) is result @@ -61,10 +61,7 @@ class TestLookupDict: assert repr(self.lookup_dict) == "" get_item_parameters = pytest.mark.parametrize( - 'key, value', ( - ('bad_gateway', 502), - ('not_a_key', None) - ) + 'key, value', (('bad_gateway', 502), ('not_a_key', None)) ) @get_item_parameters diff --git a/tests/test_testserver.py b/tests/test_testserver.py index 3c770759..caf1f938 100644 --- a/tests/test_testserver.py +++ b/tests/test_testserver.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - import threading import socket import time @@ -34,9 +33,7 @@ class TestTestServer: with Server.basic_response_server() as (host, port): sock = socket.socket() sock.connect((host, port)) - sock.close() - with pytest.raises(socket.error): new_sock = socket.socket() new_sock.connect((host, port)) @@ -44,14 +41,10 @@ class TestTestServer: def test_text_response(self): """the text_response_server sends the given text""" server = Server.text_response_server( - "HTTP/1.1 200 OK\r\n" + - "Content-Length: 6\r\n" + - "\r\nroflol" + "HTTP/1.1 200 OK\r\n" + "Content-Length: 6\r\n" + "\r\nroflol" ) - with server as (host, port): r = requests.get('http://{0}:{1}'.format(host, port)) - assert r.status_code == 200 assert r.text == u'roflol' assert r.headers['Content-Length'] == '6' @@ -67,8 +60,9 @@ class TestTestServer: def test_basic_waiting_server(self): """the server waits for the block_server event to be set before closing""" block_server = threading.Event() - - with Server.basic_response_server(wait_to_close_event=block_server) as (host, port): + with Server.basic_response_server(wait_to_close_event=block_server) as ( + host, port + ): sock = socket.socket() sock.connect((host, port)) sock.sendall(b'send something') @@ -79,15 +73,12 @@ class TestTestServer: def test_multiple_requests(self): """multiple requests can be served""" requests_to_handle = 5 - server = Server.basic_response_server(requests_to_handle=requests_to_handle) - with server as (host, port): server_url = 'http://{0}:{1}'.format(host, port) for _ in range(requests_to_handle): r = requests.get(server_url) assert r.status_code == 200 - # the (n+1)th request fails with pytest.raises(requests.exceptions.ConnectionError): r = requests.get(server_url) @@ -99,47 +90,39 @@ class TestTestServer: server = Server.basic_response_server(requests_to_handle=2) first_request = b'put your hands up in the air' second_request = b'put your hand down in the floor' - with server as address: sock1 = socket.socket() sock2 = socket.socket() - sock1.connect(address) sock1.sendall(first_request) sock1.close() - sock2.connect(address) sock2.sendall(second_request) sock2.close() - assert server.handler_results[0] == first_request assert server.handler_results[1] == second_request def test_requests_after_timeout_are_not_received(self): """the basic response handler times out when receiving requests""" server = Server.basic_response_server(request_timeout=1) - with server as address: sock = socket.socket() sock.connect(address) time.sleep(1.5) sock.sendall(b'hehehe, not received') sock.close() - assert server.handler_results[0] == b'' def test_request_recovery_with_bigger_timeout(self): """a biggest timeout can be specified""" server = Server.basic_response_server(request_timeout=3) data = b'bananadine' - with server as address: sock = socket.socket() sock.connect(address) time.sleep(1.5) sock.sendall(data) sock.close() - assert server.handler_results[0] == data def test_server_finishes_on_error(self): @@ -151,16 +134,16 @@ class TestTestServer: assert len(server.handler_results) == 0 - # if the server thread fails to finish, the test suite will hang - # and get killed by the jenkins timeout. + # if the server thread fails to finish, the test suite will hang + # and get killed by the jenkins timeout. def test_server_finishes_when_no_connections(self): """the server thread exits even if there are no connections""" server = Server.basic_response_server() with server: pass - assert len(server.handler_results) == 0 - # if the server thread fails to finish, the test suite will hang - # and get killed by the jenkins timeout. + +# if the server thread fails to finish, the test suite will hang +# and get killed by the jenkins timeout. diff --git a/tests/test_utils.py b/tests/test_utils.py index bc8528cf..7b42ca72 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - import os import copy from io import BytesIO @@ -9,16 +8,31 @@ from requests import basics from requests.cookies import RequestsCookieJar from requests.structures import CaseInsensitiveDict from requests.utils import ( - address_in_network, dotted_netmask, - get_auth_from_url, get_encoding_from_headers, - get_encodings_from_content, get_environ_proxies, - guess_filename, guess_json_utf, is_ipv4_address, - is_valid_cidr, iter_slices, parse_dict_header, - parse_header_links, prepend_scheme_if_needed, - requote_uri, select_proxy, should_bypass_proxies, super_len, - to_key_val_list, to_native_string, - unquote_header_value, unquote_unreserved, - urldefragauth, add_dict_to_cookiejar, set_environ + address_in_network, + dotted_netmask, + get_auth_from_url, + get_encoding_from_headers, + get_encodings_from_content, + get_environ_proxies, + guess_filename, + guess_json_utf, + is_ipv4_address, + is_valid_cidr, + iter_slices, + parse_dict_header, + parse_header_links, + prepend_scheme_if_needed, + requote_uri, + select_proxy, + should_bypass_proxies, + super_len, + to_key_val_list, + to_native_string, + unquote_header_value, + unquote_unreserved, + urldefragauth, + add_dict_to_cookiejar, + set_environ, ) from requests._internal_utils import unicode_is_ascii @@ -28,10 +42,8 @@ from .compat import StringIO class TestSuperLen: @pytest.mark.parametrize( - 'stream, value', ( - (StringIO.StringIO, 'Test'), - (BytesIO, b'Test') - )) + 'stream, value', ((StringIO.StringIO, 'Test'), (BytesIO, b'Test')) + ) def test_io_streams(self, stream, value): """Ensures that we properly deal with different kinds of IO streams.""" assert super_len(stream()) == 0 @@ -46,7 +58,9 @@ class TestSuperLen: @pytest.mark.parametrize('error', [IOError, OSError]) def test_super_len_handles_files_raising_weird_errors_in_tell(self, error): """If tell() raises errors, assume the cursor is at position zero.""" + class BoomFile(object): + def __len__(self): return 5 @@ -58,7 +72,9 @@ class TestSuperLen: @pytest.mark.parametrize('error', [IOError, OSError]) def test_super_len_tell_ioerror(self, error): """Ensure that if tell gives an IOError super_len doesn't fail""" + class NoLenBoomFile(object): + def tell(self): raise error() @@ -70,11 +86,7 @@ class TestSuperLen: def test_string(self): assert super_len('Test') == 4 - @pytest.mark.parametrize( - 'mode, warnings_num', ( - ('r', 1), - ('rb', 0), - )) + @pytest.mark.parametrize('mode, warnings_num', (('r', 1), ('rb', 0))) def test_file(self, tmpdir, mode, warnings_num, recwarn): file_obj = tmpdir.join('test.txt') file_obj.write('Test') @@ -83,12 +95,14 @@ class TestSuperLen: assert len(recwarn) == warnings_num def test_super_len_with__len__(self): - foo = [1,2,3,4] + foo = [1, 2, 3, 4] len_foo = super_len(foo) assert len_foo == 4 def test_super_len_with_no__len__(self): + class LenFile(object): + def __init__(self): self.len = 5 @@ -114,12 +128,14 @@ class TestSuperLen: class TestToKeyValList: @pytest.mark.parametrize( - 'value, expected', ( + 'value, expected', + ( ([('key', 'val')], [('key', 'val')]), - ((('key', 'val'), ), [('key', 'val')]), + ((('key', 'val'),), [('key', 'val')]), ({'key': 'val'}, [('key', 'val')]), - (None, None) - )) + (None, None), + ), + ) def test_valid(self, value, expected): assert to_key_val_list(value) == expected @@ -131,13 +147,15 @@ class TestToKeyValList: class TestUnquoteHeaderValue: @pytest.mark.parametrize( - 'value, expected', ( + 'value, expected', + ( (None, None), ('Test', 'Test'), ('"Test"', 'Test'), ('"Test\\\\"', 'Test\\'), ('"\\\\Comp\\Res"', '\\Comp\\Res'), - )) + ), + ) def test_valid(self, value, expected): assert unquote_header_value(value) == expected @@ -152,46 +170,48 @@ class TestGetEnvironProxies: @pytest.fixture(autouse=True, params=['no_proxy', 'NO_PROXY']) def no_proxy(self, request, monkeypatch): - monkeypatch.setenv(request.param, '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') + monkeypatch.setenv( + request.param, '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1' + ) @pytest.mark.parametrize( - 'url', ( + 'url', + ( 'http://192.168.0.1:5000/', 'http://192.168.0.1/', 'http://172.16.1.1/', 'http://172.16.1.1:5000/', 'http://localhost.localdomain:5000/v1.0/', - )) + ), + ) def test_bypass(self, url): assert get_environ_proxies(url, no_proxy=None) == {} @pytest.mark.parametrize( - 'url', ( - 'http://192.168.1.1:5000/', - 'http://192.168.1.1/', - 'http://www.requests.com/', - )) + 'url', + ('http://192.168.1.1:5000/', 'http://192.168.1.1/', 'http://www.requests.com/'), + ) def test_not_bypass(self, url): assert get_environ_proxies(url, no_proxy=None) != {} @pytest.mark.parametrize( - 'url', ( - 'http://192.168.1.1:5000/', - 'http://192.168.1.1/', - 'http://www.requests.com/', - )) + 'url', + ('http://192.168.1.1:5000/', 'http://192.168.1.1/', 'http://www.requests.com/'), + ) def test_bypass_no_proxy_keyword(self, url): no_proxy = '192.168.1.1,requests.com' assert get_environ_proxies(url, no_proxy=no_proxy) == {} @pytest.mark.parametrize( - 'url', ( + 'url', + ( 'http://192.168.0.1:5000/', 'http://192.168.0.1/', 'http://172.16.1.1/', 'http://172.16.1.1:5000/', 'http://localhost.localdomain:5000/v1.0/', - )) + ), + ) def test_not_bypass_no_proxy_keyword(self, url, monkeypatch): # This is testing that the 'no_proxy' argument overrides the # environment variable 'no_proxy' @@ -216,13 +236,15 @@ class TestIsValidCIDR: assert is_valid_cidr('192.168.1.0/24') @pytest.mark.parametrize( - 'value', ( + 'value', + ( '8.8.8.8', '192.168.1.0/a', '192.168.1.0/128', '192.168.1.0/-1', '192.168.1.999/24', - )) + ), + ) def test_invalid(self, value): assert not is_valid_cidr(value) @@ -238,17 +260,14 @@ class TestAddressInNetwork: class TestGuessFilename: - @pytest.mark.parametrize( - 'value', (1, type('Fake', (object,), {'name': 1})()), - ) + @pytest.mark.parametrize('value', (1, type('Fake', (object,), {'name': 1})())) def test_guess_filename_invalid(self, value): assert guess_filename(value) is None @pytest.mark.parametrize( - 'value, expected_type', ( - (b'value', basics.bytes), - (b'value'.decode('utf-8'), basics.str) - )) + 'value, expected_type', + ((b'value', basics.bytes), (b'value'.decode('utf-8'), basics.str)), + ) def test_guess_filename_valid(self, value, expected_type): obj = type('Fake', (object,), {'name': value})() result = guess_filename(obj) @@ -263,16 +282,13 @@ class TestContentEncodingDetection: assert not len(encodings) @pytest.mark.parametrize( - 'content', ( - # HTML5 meta charset attribute - '', - # HTML4 pragma directive - '', - # XHTML 1.x served with text/html MIME type - '', - # XHTML 1.x served as XML - '', - )) + 'content', + ('', '', '', ''), + # HTML5 meta charset attribute + # HTML4 pragma directive + # XHTML 1.x served with text/html MIME type + # XHTML 1.x served as XML + ) def test_pragmas(self, content): encodings = get_encodings_from_content(content) assert len(encodings) == 1 @@ -283,17 +299,26 @@ class TestContentEncodingDetection: - '''.strip() + '''.strip( + ) assert get_encodings_from_content(content) == ['HTML5', 'HTML4', 'XML'] class TestGuessJSONUTF: @pytest.mark.parametrize( - 'encoding', ( - 'utf-32', 'utf-8-sig', 'utf-16', 'utf-8', 'utf-16-be', 'utf-16-le', - 'utf-32-be', 'utf-32-le' - )) + 'encoding', + ( + 'utf-32', + 'utf-8-sig', + 'utf-16', + 'utf-8', + 'utf-16-be', + 'utf-16-le', + 'utf-32-be', + 'utf-32-le', + ), + ) def test_encoded(self, encoding): data = '{}'.encode(encoding) assert guess_json_utf(data) == encoding @@ -302,12 +327,14 @@ class TestGuessJSONUTF: assert guess_json_utf(b'\x00\x00\x00\x00') is None @pytest.mark.parametrize( - ('encoding', 'expected'), ( + ('encoding', 'expected'), + ( ('utf-16-be', 'utf-16'), ('utf-16-le', 'utf-16'), ('utf-32-be', 'utf-32'), - ('utf-32-le', 'utf-32') - )) + ('utf-32-le', 'utf-32'), + ), + ) def test_guess_by_bom(self, encoding, expected): data = u'\ufeff{}'.encode(encoding) assert guess_json_utf(data) == expected @@ -319,156 +346,119 @@ ENCODED_PASSWORD = basics.quote(PASSWORD, '') @pytest.mark.parametrize( - 'url, auth', ( + 'url, auth', + ( ( - 'http://' + ENCODED_USER + ':' + ENCODED_PASSWORD + '@' + + 'http://' + + ENCODED_USER + + ':' + + ENCODED_PASSWORD + + '@' + 'request.com/url.html#test', - (USER, PASSWORD) - ), - ( - 'http://user:pass@complex.url.com/path?query=yes', - ('user', 'pass') + (USER, PASSWORD), ), + ('http://user:pass@complex.url.com/path?query=yes', ('user', 'pass')), ( 'http://user:pass%20pass@complex.url.com/path?query=yes', - ('user', 'pass pass') - ), - ( - 'http://user:pass pass@complex.url.com/path?query=yes', - ('user', 'pass pass') + ('user', 'pass pass'), ), + ('http://user:pass pass@complex.url.com/path?query=yes', ('user', 'pass pass')), ( 'http://user%25user:pass@complex.url.com/path?query=yes', - ('user%user', 'pass') + ('user%user', 'pass'), ), ( 'http://user:pass%23pass@complex.url.com/path?query=yes', - ('user', 'pass#pass') + ('user', 'pass#pass'), ), - ( - 'http://complex.url.com/path?query=yes', - ('', '') - ), - )) + ('http://complex.url.com/path?query=yes', ('', '')), + ), +) def test_get_auth_from_url(url, auth): assert get_auth_from_url(url) == auth @pytest.mark.parametrize( - 'uri, expected', ( - ( - # Ensure requoting doesn't break expectations - 'http://example.com/fiz?buz=%25ppicture', - 'http://example.com/fiz?buz=%25ppicture', - ), - ( - # Ensure we handle unquoted percent signs in redirects - 'http://example.com/fiz?buz=%ppicture', - 'http://example.com/fiz?buz=%25ppicture', - ), - )) + 'uri, expected', + (('http://example.com/fiz?buz=%25ppicture', 'http://example.com/fiz?buz=%25ppicture'), ('http://example.com/fiz?buz=%ppicture', 'http://example.com/fiz?buz=%25ppicture')), + # Ensure requoting doesn't break expectations + # Ensure we handle unquoted percent signs in redirects +) def test_requote_uri_with_unquoted_percents(uri, expected): """See: https://github.com/requests/requests/issues/2356""" assert requote_uri(uri) == expected @pytest.mark.parametrize( - 'uri, expected', ( - ( - # Illegal bytes - 'http://example.com/?a=%--', - 'http://example.com/?a=%--', - ), - ( - # Reserved characters - 'http://example.com/?a=%300', - 'http://example.com/?a=00', - ) - )) + 'uri, expected', + (('http://example.com/?a=%--', 'http://example.com/?a=%--'), ('http://example.com/?a=%300', 'http://example.com/?a=00')), + # Illegal bytes + # Reserved characters +) def test_unquote_unreserved(uri, expected): assert unquote_unreserved(uri) == expected @pytest.mark.parametrize( - 'mask, expected', ( - (8, '255.0.0.0'), - (24, '255.255.255.0'), - (25, '255.255.255.128'), - )) + 'mask, expected', ((8, '255.0.0.0'), (24, '255.255.255.0'), (25, '255.255.255.128')) +) def test_dotted_netmask(mask, expected): assert dotted_netmask(mask) == expected -http_proxies = {'http': 'http://http.proxy', - 'http://some.host': 'http://some.host.proxy'} -all_proxies = {'all': 'socks5://http.proxy', - 'all://some.host': 'socks5://some.host.proxy'} -mixed_proxies = {'http': 'http://http.proxy', - 'http://some.host': 'http://some.host.proxy', - 'all': 'socks5://http.proxy'} +http_proxies = { + 'http': 'http://http.proxy', 'http://some.host': 'http://some.host.proxy' +} +all_proxies = { + 'all': 'socks5://http.proxy', 'all://some.host': 'socks5://some.host.proxy' +} +mixed_proxies = { + 'http': 'http://http.proxy', + 'http://some.host': 'http://some.host.proxy', + 'all': 'socks5://http.proxy', +} + + @pytest.mark.parametrize( - 'url, expected, proxies', ( - ('hTTp://u:p@Some.Host/path', 'http://some.host.proxy', http_proxies), - ('hTTp://u:p@Other.Host/path', 'http://http.proxy', http_proxies), - ('hTTp:///path', 'http://http.proxy', http_proxies), - ('hTTps://Other.Host', None, http_proxies), - ('file:///etc/motd', None, http_proxies), - - ('hTTp://u:p@Some.Host/path', 'socks5://some.host.proxy', all_proxies), - ('hTTp://u:p@Other.Host/path', 'socks5://http.proxy', all_proxies), - ('hTTp:///path', 'socks5://http.proxy', all_proxies), - ('hTTps://Other.Host', 'socks5://http.proxy', all_proxies), - - ('http://u:p@other.host/path', 'http://http.proxy', mixed_proxies), - ('http://u:p@some.host/path', 'http://some.host.proxy', mixed_proxies), - ('https://u:p@other.host/path', 'socks5://http.proxy', mixed_proxies), - ('https://u:p@some.host/path', 'socks5://http.proxy', mixed_proxies), - ('https://', 'socks5://http.proxy', mixed_proxies), - # XXX: unsure whether this is reasonable behavior - ('file:///etc/motd', 'socks5://http.proxy', all_proxies), - )) + 'url, expected, proxies', + (('hTTp://u:p@Some.Host/path', 'http://some.host.proxy', http_proxies), ('hTTp://u:p@Other.Host/path', 'http://http.proxy', http_proxies), ('hTTp:///path', 'http://http.proxy', http_proxies), ('hTTps://Other.Host', None, http_proxies), ('file:///etc/motd', None, http_proxies), ('hTTp://u:p@Some.Host/path', 'socks5://some.host.proxy', all_proxies), ('hTTp://u:p@Other.Host/path', 'socks5://http.proxy', all_proxies), ('hTTp:///path', 'socks5://http.proxy', all_proxies), ('hTTps://Other.Host', 'socks5://http.proxy', all_proxies), ('http://u:p@other.host/path', 'http://http.proxy', mixed_proxies), ('http://u:p@some.host/path', 'http://some.host.proxy', mixed_proxies), ('https://u:p@other.host/path', 'socks5://http.proxy', mixed_proxies), ('https://u:p@some.host/path', 'socks5://http.proxy', mixed_proxies), ('https://', 'socks5://http.proxy', mixed_proxies), ('file:///etc/motd', 'socks5://http.proxy', all_proxies)), + # XXX: unsure whether this is reasonable behavior +) def test_select_proxies(url, expected, proxies): """Make sure we can select per-host proxies correctly.""" assert select_proxy(url, proxies) == expected @pytest.mark.parametrize( - 'value, expected', ( + 'value, expected', + ( ('foo="is a fish", bar="as well"', {'foo': 'is a fish', 'bar': 'as well'}), - ('key_without_value', {'key_without_value': None}) - )) + ('key_without_value', {'key_without_value': None}), + ), +) def test_parse_dict_header(value, expected): assert parse_dict_header(value) == expected @pytest.mark.parametrize( - 'value, expected', ( - ( - CaseInsensitiveDict(), - None - ), + 'value, expected', + ( + (CaseInsensitiveDict(), None), ( CaseInsensitiveDict({'content-type': 'application/json; charset=utf-8'}), - 'utf-8' + 'utf-8', ), - ( - CaseInsensitiveDict({'content-type': 'text/plain'}), - 'ISO-8859-1' - ), - )) + (CaseInsensitiveDict({'content-type': 'text/plain'}), 'ISO-8859-1'), + ), +) def test_get_encoding_from_headers(value, expected): assert get_encoding_from_headers(value) == expected @pytest.mark.parametrize( - 'value, length', ( - ('', 0), - ('T', 1), - ('Test', 4), - ('Cont', 0), - ('Other', -5), - ('Content', None), - )) + 'value, length', + (('', 0), ('T', 1), ('Test', 4), ('Cont', 0), ('Other', -5), ('Content', None)), +) def test_iter_slices(value, length): if length is None or (length <= 0 and len(value) > 0): # Reads all content at once @@ -478,127 +468,119 @@ def test_iter_slices(value, length): @pytest.mark.parametrize( - 'value, expected', ( + 'value, expected', + ( ( '; rel=front; type="image/jpeg"', - [{'url': 'http:/.../front.jpeg', 'rel': 'front', 'type': 'image/jpeg'}] - ), - ( - '', - [{'url': 'http:/.../front.jpeg'}] - ), - ( - ';', - [{'url': 'http:/.../front.jpeg'}] + [{'url': 'http:/.../front.jpeg', 'rel': 'front', 'type': 'image/jpeg'}], ), + ('', [{'url': 'http:/.../front.jpeg'}]), + (';', [{'url': 'http:/.../front.jpeg'}]), ( '; type="image/jpeg",;', [ {'url': 'http:/.../front.jpeg', 'type': 'image/jpeg'}, - {'url': 'http://.../back.jpeg'} - ] + {'url': 'http://.../back.jpeg'}, + ], ), - ( - '', - [] - ), - )) + ('', []), + ), +) def test_parse_header_links(value, expected): assert parse_header_links(value) == expected @pytest.mark.parametrize( - 'value, expected', ( + 'value, expected', + ( ('example.com/path', 'http://example.com/path'), ('//example.com/path', 'http://example.com/path'), - )) + ), +) def test_prepend_scheme_if_needed(value, expected): assert prepend_scheme_if_needed(value, 'http') == expected -@pytest.mark.parametrize( - 'value, expected', ( - ('T', 'T'), - (b'T', 'T'), - (u'T', 'T'), - )) +@pytest.mark.parametrize('value, expected', (('T', 'T'), (b'T', 'T'), (u'T', 'T'))) def test_to_native_string(value, expected): assert to_native_string(value) == expected @pytest.mark.parametrize( - 'url, expected', ( + 'url, expected', + ( ('http://u:p@example.com/path?a=1#test', 'http://example.com/path?a=1'), ('http://example.com/path', 'http://example.com/path'), ('//u:p@example.com/path', '//example.com/path'), ('//example.com/path', '//example.com/path'), ('example.com/path', '//example.com/path'), ('scheme:u:p@example.com/path', 'scheme://example.com/path'), - )) + ), +) def test_urldefragauth(url, expected): assert urldefragauth(url) == expected @pytest.mark.parametrize( - 'url, expected', ( - ('http://192.168.0.1:5000/', True), - ('http://192.168.0.1/', True), - ('http://172.16.1.1/', True), - ('http://172.16.1.1:5000/', True), - ('http://localhost.localdomain:5000/v1.0/', True), - ('http://172.16.1.12/', False), - ('http://172.16.1.12:5000/', False), - ('http://google.com:5000/v1.0/', False), - )) + 'url, expected', + ( + ('http://192.168.0.1:5000/', True), + ('http://192.168.0.1/', True), + ('http://172.16.1.1/', True), + ('http://172.16.1.1:5000/', True), + ('http://localhost.localdomain:5000/v1.0/', True), + ('http://172.16.1.12/', False), + ('http://172.16.1.12:5000/', False), + ('http://google.com:5000/v1.0/', False), + ), +) def test_should_bypass_proxies(url, expected, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not """ - monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') - monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') + monkeypatch.setenv( + 'no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1' + ) + monkeypatch.setenv( + 'NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1' + ) assert should_bypass_proxies(url, no_proxy=None) == expected @pytest.mark.parametrize( - 'cookiejar', ( - basics.cookielib.CookieJar(), - RequestsCookieJar() - )) + 'cookiejar', (basics.cookielib.CookieJar(), RequestsCookieJar()) +) def test_add_dict_to_cookiejar(cookiejar): """Ensure add_dict_to_cookiejar works for non-RequestsCookieJar CookieJars """ - cookiedict = {'test': 'cookies', - 'good': 'cookies'} + cookiedict = {'test': 'cookies', 'good': 'cookies'} cj = add_dict_to_cookiejar(cookiejar, cookiedict) cookies = {cookie.name: cookie.value for cookie in cj} assert cookiedict == cookies @pytest.mark.parametrize( - 'value, expected', ( - (u'test', True), - (u'æíöû', False), - (u'ジェーピーニック', False), - ) + 'value, expected', ((u'test', True), (u'æíöû', False), (u'ジェーピーニック', False)) ) def test_unicode_is_ascii(value, expected): assert unicode_is_ascii(value) is expected @pytest.mark.parametrize( - 'url, expected', ( - ('http://192.168.0.1:5000/', True), - ('http://192.168.0.1/', True), - ('http://172.16.1.1/', True), - ('http://172.16.1.1:5000/', True), - ('http://localhost.localdomain:5000/v1.0/', True), - ('http://172.16.1.12/', False), - ('http://172.16.1.12:5000/', False), - ('http://google.com:5000/v1.0/', False), - )) -def test_should_bypass_proxies_no_proxy( - url, expected, monkeypatch): + 'url, expected', + ( + ('http://192.168.0.1:5000/', True), + ('http://192.168.0.1/', True), + ('http://172.16.1.1/', True), + ('http://172.16.1.1:5000/', True), + ('http://localhost.localdomain:5000/v1.0/', True), + ('http://172.16.1.12/', False), + ('http://172.16.1.12:5000/', False), + ('http://google.com:5000/v1.0/', False), + ), +) +def test_should_bypass_proxies_no_proxy(url, expected, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not using the 'no_proxy' argument """ @@ -609,20 +591,21 @@ def test_should_bypass_proxies_no_proxy( @pytest.mark.skipif(os.name != 'nt', reason='Test only on Windows') @pytest.mark.parametrize( - 'url, expected, override', ( - ('http://192.168.0.1:5000/', True, None), - ('http://192.168.0.1/', True, None), - ('http://172.16.1.1/', True, None), - ('http://172.16.1.1:5000/', True, None), - ('http://localhost.localdomain:5000/v1.0/', True, None), - ('http://172.16.1.22/', False, None), - ('http://172.16.1.22:5000/', False, None), - ('http://google.com:5000/v1.0/', False, None), - ('http://mylocalhostname:5000/v1.0/', True, ''), - ('http://192.168.0.1/', False, ''), - )) -def test_should_bypass_proxies_win_registry(url, expected, override, - monkeypatch): + 'url, expected, override', + ( + ('http://192.168.0.1:5000/', True, None), + ('http://192.168.0.1/', True, None), + ('http://172.16.1.1/', True, None), + ('http://172.16.1.1:5000/', True, None), + ('http://localhost.localdomain:5000/v1.0/', True, None), + ('http://172.16.1.22/', False, None), + ('http://172.16.1.22:5000/', False, None), + ('http://google.com:5000/v1.0/', False, None), + ('http://mylocalhostname:5000/v1.0/', True, ''), + ('http://192.168.0.1/', False, ''), + ), +) +def test_should_bypass_proxies_win_registry(url, expected, override, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not with Windows registry settings """ @@ -634,6 +617,7 @@ def test_should_bypass_proxies_win_registry(url, expected, override, import _winreg as winreg class RegHandle: + def Close(self): pass @@ -646,6 +630,7 @@ def test_should_bypass_proxies_win_registry(url, expected, override, if key is ie_settings: if value_name == 'ProxyEnable': return [1] + elif value_name == 'ProxyOverride': return [override] @@ -659,18 +644,19 @@ def test_should_bypass_proxies_win_registry(url, expected, override, @pytest.mark.parametrize( - 'env_name, value', ( - ('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain'), - ('no_proxy', None), - ('a_new_key', '192.168.0.0/24,127.0.0.1,localhost.localdomain'), - ('a_new_key', None), - )) + 'env_name, value', + ( + ('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain'), + ('no_proxy', None), + ('a_new_key', '192.168.0.0/24,127.0.0.1,localhost.localdomain'), + ('a_new_key', None), + ), +) def test_set_environ(env_name, value): """Tests set_environ will set environ values and will restore the environ.""" environ_copy = copy.deepcopy(os.environ) with set_environ(env_name, value): assert os.environ.get(env_name) == value - assert os.environ == environ_copy diff --git a/tests/testserver/server.py b/tests/testserver/server.py index 6a1dcaa5..e8ff4ab6 100644 --- a/tests/testserver/server.py +++ b/tests/testserver/server.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - import threading import socket import select @@ -8,7 +7,6 @@ import select def consume_socket_content(sock, timeout=0.5): chunks = 65536 content = b'' - while True: more_to_read = select.select([sock], [], [], timeout)[0] if not more_to_read: @@ -19,7 +17,6 @@ def consume_socket_content(sock, timeout=0.5): break content += new_content - return content @@ -27,37 +24,38 @@ class Server(threading.Thread): """Dummy server using for unit testing""" WAIT_EVENT_TIMEOUT = 5 - def __init__(self, handler=None, host='localhost', port=0, requests_to_handle=1, wait_to_close_event=None): + def __init__( + self, + handler=None, + host='localhost', + port=0, + requests_to_handle=1, + wait_to_close_event=None, + ): super(Server, self).__init__() - self.handler = handler or consume_socket_content self.handler_results = [] - self.host = host self.port = port self.requests_to_handle = requests_to_handle - self.wait_to_close_event = wait_to_close_event self.ready_event = threading.Event() self.stop_event = threading.Event() @classmethod def text_response_server(cls, text, request_timeout=0.5, **kwargs): + def text_response_handler(sock): request_content = consume_socket_content(sock, timeout=request_timeout) sock.send(text.encode('utf-8')) - return request_content - return Server(text_response_handler, **kwargs) @classmethod def basic_response_server(cls, **kwargs): return cls.text_response_server( - "HTTP/1.1 200 OK\r\n" + - "Content-Length: 0\r\n\r\n", - **kwargs + "HTTP/1.1 200 OK\r\n" + "Content-Length: 0\r\n\r\n", **kwargs ) def run(self): @@ -67,11 +65,10 @@ class Server(threading.Thread): self.port = self.server_sock.getsockname()[1] self.ready_event.set() self._handle_requests() - if self.wait_to_close_event: self.wait_to_close_event.wait(self.WAIT_EVENT_TIMEOUT) finally: - self.ready_event.set() # just in case of exception + self.ready_event.set() # just in case of exception self._close_server_sock_ignore_errors() self.stop_event.set() @@ -94,16 +91,18 @@ class Server(threading.Thread): break handler_result = self.handler(sock) - self.handler_results.append(handler_result) def _accept_connection(self): try: - ready, _, _ = select.select([self.server_sock], [], [], self.WAIT_EVENT_TIMEOUT) + ready, _, _ = select.select( + [self.server_sock], [], [], self.WAIT_EVENT_TIMEOUT + ) if not ready: return None return self.server_sock.accept()[0] + except (select.error, socket.error): return None @@ -120,8 +119,7 @@ class Server(threading.Thread): # avoid server from waiting for event timeouts # if an exception is found in the main thread self.wait_to_close_event.set() - # ensure server thread doesn't get stuck waiting for connections self._close_server_sock_ignore_errors() self.join() - return False # allow exceptions to propagate + return False # allow exceptions to propagate diff --git a/tests/utils.py b/tests/utils.py index 9b797fd4..b19b3554 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - import contextlib import os @@ -14,6 +13,7 @@ def override_environ(**kwargs): os.environ[key] = value try: yield + finally: os.environ.clear() os.environ.update(save_env)