From 27eb54a20a6f21cfe7138d4a26a8f41f5b2ced4b Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 28 Jan 2014 20:13:57 -0600 Subject: [PATCH 01/28] Move creation of attributes to RequestException Pass request objects in HTTPAdapter --- requests/adapters.py | 8 ++++---- requests/exceptions.py | 15 ++++++++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index dd10e959..ca462232 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -372,19 +372,19 @@ class HTTPAdapter(BaseAdapter): conn._put_conn(low_conn) except socket.error as sockerr: - raise ConnectionError(sockerr) + raise ConnectionError(sockerr, request=request) except MaxRetryError as e: - raise ConnectionError(e) + raise ConnectionError(e, request=request) except _ProxyError as e: raise ProxyError(e) except (_SSLError, _HTTPError) as e: if isinstance(e, _SSLError): - raise SSLError(e) + raise SSLError(e, request=request) elif isinstance(e, TimeoutError): - raise Timeout(e) + raise Timeout(e, request=request) else: raise diff --git a/requests/exceptions.py b/requests/exceptions.py index cd3c7600..7c291f2b 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -14,15 +14,20 @@ class RequestException(IOError): """There was an ambiguous exception that occurred while handling your request.""" + def __init__(self, *args, **kwargs): + """ + Initialize RequestException with `request` and `response` objects. + """ + self.response = kwargs.pop('response', None) + self.request = kwargs.pop('request', None) + if self.response and not self.request: + self.request = self.response.request + super(RequestException, self).__init__(*args, **kwargs) + class HTTPError(RequestException): """An HTTP error occurred.""" - def __init__(self, *args, **kwargs): - """ Initializes HTTPError with optional `response` object. """ - self.response = kwargs.pop('response', None) - super(HTTPError, self).__init__(*args, **kwargs) - class ConnectionError(RequestException): """A Connection error occurred.""" From c2fab5b4ca2f8dcdc76fabb1851bd268f0d41889 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 28 Jan 2014 20:24:41 -0600 Subject: [PATCH 02/28] Avoid having to fix tests --- requests/exceptions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requests/exceptions.py b/requests/exceptions.py index 7c291f2b..c8ec089f 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -18,9 +18,10 @@ class RequestException(IOError): """ Initialize RequestException with `request` and `response` objects. """ - self.response = kwargs.pop('response', None) + response = kwargs.pop('response', None) + self.response = response self.request = kwargs.pop('request', None) - if self.response and not self.request: + if response and not self.request and hasattr(response, 'request'): self.request = self.response.request super(RequestException, self).__init__(*args, **kwargs) From 07e0a6198d132fee147806105a6ff05e541d3ba2 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 28 Jan 2014 20:39:09 -0600 Subject: [PATCH 03/28] Explicitly check for None --- requests/exceptions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requests/exceptions.py b/requests/exceptions.py index c8ec089f..a4ee9d63 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -21,7 +21,8 @@ class RequestException(IOError): response = kwargs.pop('response', None) self.response = response self.request = kwargs.pop('request', None) - if response and not self.request and hasattr(response, 'request'): + if (response is not None and not self.request and + hasattr(response, 'request')): self.request = self.response.request super(RequestException, self).__init__(*args, **kwargs) From b5b8198fd1e872471f4523e0b450dfee33072242 Mon Sep 17 00:00:00 2001 From: Aaron Iles Date: Sun, 2 Feb 2014 20:14:05 +1100 Subject: [PATCH 04/28] Document requirements for SNI support on Python2 A section for Request's advanced usage guide on what Server Name Indication is, its purpose, and how to enable it on Python2. --- docs/user/advanced.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index f5e8e59f..9df9f9b1 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -179,6 +179,31 @@ If you specify a wrong path or an invalid cert:: >>> requests.get('https://kennethreitz.com', cert='/wrong_path/server.pem') SSLError: [Errno 336265225] _ssl.c:347: error:140B0009:SSL routines:SSL_CTX_use_PrivateKey_file:PEM lib +Server Name Indication +---------------------- + +`Server Name Indication`_, or SNI, is an official extension to SSL where the +client tells the server what hostname it is contacting. This enables `virtual +hosting`_ on SSL protected sites. + +Python3's SSL module includes native support for SNI. This support has not been +back ported to Python2. However, Requests will enable SNI support on Python2 if +the following packages are installed: + +* `pyOpenSSL`_, a Python wrapper module around the OpenSSL library. +* `ndg-httpsclient`_, enhanced HTTPS support for httplib and urllib2. +* `pyasn1`_, ASN.1 types and codecs. + +When these packages are installed, Requests will automatically indicate to the +server what hostname is being contacted. This allows the server to return the +correct server certificate for SSL certificate verification. + +.. _`Server Name Indication`: https://en.wikipedia.org/wiki/Server_Name_Indication +.. _`virtual hosting`: https://en.wikipedia.org/wiki/Virtual_hosting +.. _`pyOpenSSL`: https://pypi.python.org/pypi/pyOpenSSL +.. _`ndg-httpsclient`: https://pypi.python.org/pypi/ndg-httpsclient +.. _`pyasn1`: https://pypi.python.org/pypi/pyasn1 + Body Content Workflow --------------------- From 8a0bae45c2efa8706c76d8af925d0dcd727cfc32 Mon Sep 17 00:00:00 2001 From: Aaron Iles Date: Sun, 2 Feb 2014 22:19:10 +1100 Subject: [PATCH 05/28] Move SNI documentation to FAQ Relocate documentation on Server-Name-Indication from the advanced section to the frequently asked questions. This is minus details on enabling SNI on Python2, which is instead captured by linking to Stack Overflow. --- docs/community/faq.rst | 22 ++++++++++++++++++++++ docs/user/advanced.rst | 27 ++------------------------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/docs/community/faq.rst b/docs/community/faq.rst index edbf9b70..84c6b80a 100644 --- a/docs/community/faq.rst +++ b/docs/community/faq.rst @@ -60,3 +60,25 @@ supported: * Python 3.2 * Python 3.3 * PyPy 1.9 + +What are "hostname doesn't match" errors? +----------------------------------------- + +These errors occur when :ref:`SSL certificate verification ` +fails to match the certificate the server responds with to the hostname +Requests thinks it's contacting. If you're certain the server's SSL setup is +correct (for example, because you can visit the site with your browser) a +possible explanation is Request's is lacking Server-Name-Indication. + +`Server-Name-Indication`_, or SNI, is an official extension to SSL where the +client tells the server what hostname it is contacting. This enables `virtual +hosting`_ on SSL protected sites, the server being to able to respond with a +certificate appropriate for the hostname the client is contacting. + +Python3's SSL module includes native support for SNI. This support has not been +back ported to Python2. For information on using SNI with Requests on Python2 +refer to this `Stack Overflow answer`_. + +.. _`Server-Name-Indication`: https://en.wikipedia.org/wiki/Server_Name_Indication +.. _`virtual hosting`: https://en.wikipedia.org/wiki/Virtual_hosting +.. _`Stack Overflow answer`: https://stackoverflow.com/questions/18578439/using-requests-with-tls-doesnt-give-sni-support/18579484#18579484 diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 9df9f9b1..80c1e6ae 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -145,6 +145,8 @@ applied, replace the call to :meth:`Request.prepare() print(resp.status_code) +.. _verification: + SSL Cert Verification --------------------- @@ -179,31 +181,6 @@ If you specify a wrong path or an invalid cert:: >>> requests.get('https://kennethreitz.com', cert='/wrong_path/server.pem') SSLError: [Errno 336265225] _ssl.c:347: error:140B0009:SSL routines:SSL_CTX_use_PrivateKey_file:PEM lib -Server Name Indication ----------------------- - -`Server Name Indication`_, or SNI, is an official extension to SSL where the -client tells the server what hostname it is contacting. This enables `virtual -hosting`_ on SSL protected sites. - -Python3's SSL module includes native support for SNI. This support has not been -back ported to Python2. However, Requests will enable SNI support on Python2 if -the following packages are installed: - -* `pyOpenSSL`_, a Python wrapper module around the OpenSSL library. -* `ndg-httpsclient`_, enhanced HTTPS support for httplib and urllib2. -* `pyasn1`_, ASN.1 types and codecs. - -When these packages are installed, Requests will automatically indicate to the -server what hostname is being contacted. This allows the server to return the -correct server certificate for SSL certificate verification. - -.. _`Server Name Indication`: https://en.wikipedia.org/wiki/Server_Name_Indication -.. _`virtual hosting`: https://en.wikipedia.org/wiki/Virtual_hosting -.. _`pyOpenSSL`: https://pypi.python.org/pypi/pyOpenSSL -.. _`ndg-httpsclient`: https://pypi.python.org/pypi/ndg-httpsclient -.. _`pyasn1`: https://pypi.python.org/pypi/pyasn1 - Body Content Workflow --------------------- From 5ee8b348ebab9a7c427a87355dd089c83ee74be9 Mon Sep 17 00:00:00 2001 From: Martijn Pieters Date: Mon, 3 Feb 2014 12:00:14 +0000 Subject: [PATCH 06/28] Reinstate falling back to self.text for JSON responses A JSON response that has no encoding specified will be decoded with a detected UTF codec (compliant with the JSON RFC), but if that fails, we guessed wrong and need to fall back to charade character detection (via `self.text`). Kenneth removed this functionality (by accident?) in 1451ba0c6d395c41f86da35036fa361c3a41bc90, this reinstates it again and adds a log warning. Fixes #1674 --- requests/models.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index 34dce181..2adc5492 100644 --- a/requests/models.py +++ b/requests/models.py @@ -725,11 +725,20 @@ class Response(object): if not self.encoding and len(self.content) > 3: # No encoding set. JSON RFC 4627 section 3 states we should expect # UTF-8, -16 or -32. Detect which one to use; If the detection or - # decoding fails, fall back to `self.text` (using chardet to make + # decoding fails, fall back to `self.text` (using charade to make # a best guess). encoding = guess_json_utf(self.content) if encoding is not None: - return json.loads(self.content.decode(encoding), **kwargs) + try: + return json.loads(self.content.decode(encoding), **kwargs) + except UnicodeDecodeError: + # Wrong UTF codec detected; usually because it's not UTF-8 + # but some other 8-bit codec. This is an RFC violation, + # and the server didn't bother to tell us what codec *was* + # used. + pass + log.warn('No encoding specified for JSON response, and no ' + 'UTF codec detected. Falling back to charade best guess.') return json.loads(self.text, **kwargs) @property From f35838beb6a84b1ece6e775570f6dd668dcd8da0 Mon Sep 17 00:00:00 2001 From: Martijn Pieters Date: Mon, 3 Feb 2014 13:37:57 +0000 Subject: [PATCH 07/28] Remove logging and charade mention. Logging is Not Allowed, so out it goes. --- requests/models.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index 2adc5492..27b73899 100644 --- a/requests/models.py +++ b/requests/models.py @@ -725,7 +725,7 @@ class Response(object): if not self.encoding and len(self.content) > 3: # No encoding set. JSON RFC 4627 section 3 states we should expect # UTF-8, -16 or -32. Detect which one to use; If the detection or - # decoding fails, fall back to `self.text` (using charade to make + # decoding fails, fall back to `self.text` (using chardet to make # a best guess). encoding = guess_json_utf(self.content) if encoding is not None: @@ -737,8 +737,6 @@ class Response(object): # and the server didn't bother to tell us what codec *was* # used. pass - log.warn('No encoding specified for JSON response, and no ' - 'UTF codec detected. Falling back to charade best guess.') return json.loads(self.text, **kwargs) @property From c5b6a107eb9af0fd6a1a48c7bf0276fbf71e9157 Mon Sep 17 00:00:00 2001 From: Martijn Pieters Date: Mon, 3 Feb 2014 13:39:21 +0000 Subject: [PATCH 08/28] One last Charade reference to remove here. --- requests/models.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index 014ca789..6270a691 100644 --- a/requests/models.py +++ b/requests/models.py @@ -593,8 +593,7 @@ class Response(object): @property def apparent_encoding(self): - """The apparent encoding, provided by the lovely Charade library - (Thanks, Ian!).""" + """The apparent encoding, provided by the chardet library""" return chardet.detect(self.content)['encoding'] def iter_content(self, chunk_size=1, decode_unicode=False): From 488b90f5acfafe65b428765aea70ebebd4afe58a Mon Sep 17 00:00:00 2001 From: Martijn Pieters Date: Mon, 3 Feb 2014 13:43:59 +0000 Subject: [PATCH 09/28] Remove unused loggers. --- requests/auth.py | 3 --- requests/models.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/requests/auth.py b/requests/auth.py index 6664cd80..9f831b7a 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -11,7 +11,6 @@ import os import re import time import hashlib -import logging from base64 import b64encode @@ -19,8 +18,6 @@ from .compat import urlparse, str from .cookies import extract_cookies_to_jar from .utils import parse_dict_header -log = logging.getLogger(__name__) - CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded' CONTENT_TYPE_MULTI_PART = 'multipart/form-data' diff --git a/requests/models.py b/requests/models.py index 014ca789..49797a7f 100644 --- a/requests/models.py +++ b/requests/models.py @@ -8,7 +8,6 @@ This module contains the primary objects that power Requests. """ import collections -import logging import datetime from io import BytesIO, UnsupportedOperation @@ -35,8 +34,6 @@ from .compat import ( CONTENT_CHUNK_SIZE = 10 * 1024 ITER_CHUNK_SIZE = 512 -log = logging.getLogger(__name__) - class RequestEncodingMixin(object): @property From 98b76f4b988145a98be7d865da407e150aadc165 Mon Sep 17 00:00:00 2001 From: Aaron Iles Date: Tue, 4 Feb 2014 21:08:11 +1100 Subject: [PATCH 10/28] State early in SNI discussion Python2 limitation Reference at the earliest opportunity that Server-Name-Indication is a limitation of Python 2.6 and 2.7. Avoid describing it as a Requests issue. --- docs/community/faq.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/community/faq.rst b/docs/community/faq.rst index 84c6b80a..4e792eca 100644 --- a/docs/community/faq.rst +++ b/docs/community/faq.rst @@ -67,8 +67,9 @@ What are "hostname doesn't match" errors? These errors occur when :ref:`SSL certificate verification ` fails to match the certificate the server responds with to the hostname Requests thinks it's contacting. If you're certain the server's SSL setup is -correct (for example, because you can visit the site with your browser) a -possible explanation is Request's is lacking Server-Name-Indication. +correct (for example, because you can visit the site with your browser) and +you're using Python 2.6 or 2.7, a possible explanation is that you need +Server-Name-Indication. `Server-Name-Indication`_, or SNI, is an official extension to SSL where the client tells the server what hostname it is contacting. This enables `virtual From 3443c177a3b8279b6c5b5ce9c1643689d8983521 Mon Sep 17 00:00:00 2001 From: Martijn Pieters Date: Wed, 5 Feb 2014 17:29:09 +0000 Subject: [PATCH 11/28] Document the `Response.reason` attribute. Made `.status_code` and `.reason` consistent with one another, adding some examples. Addresses #1225. --- requests/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 6270a691..78832054 100644 --- a/requests/models.py +++ b/requests/models.py @@ -517,7 +517,7 @@ class Response(object): self._content = False self._content_consumed = False - #: Integer Code of responded HTTP Status. + #: Integer Code of responded HTTP Status, e.g. 404 or 200. self.status_code = None #: Case-insensitive Dictionary of Response Headers. @@ -541,6 +541,7 @@ class Response(object): #: up here. The list is sorted from the oldest to the most recent request. self.history = [] + #: Textual reason of responded HTTP Status, e.g. "Not Found" or "OK". self.reason = None #: A CookieJar of Cookies the server sent back. From 969195ad55f9e44ee2e1605e8561123bfed62c4e Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 11 Feb 2014 07:31:47 +0000 Subject: [PATCH 12/28] Clarify our 1.2.1 behaviour change. --- HISTORY.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 8bdb6ab1..90ea6146 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -128,6 +128,8 @@ Release History 1.2.1 (2013-05-20) ++++++++++++++++++ +- 301 and 302 redirects now change the verb to GET for all verbs, not just + POST, improving browser compatibility. - Python 3.3.2 compatibility - Always percent-encode location headers - Fix connection adapter matching to be most-specific first From 0caa2432123bab2d991e635ce558226d019d7bc7 Mon Sep 17 00:00:00 2001 From: Zack Weinberg Date: Thu, 13 Feb 2014 14:27:42 -0500 Subject: [PATCH 13/28] New Response property, .is_redirect. --- HISTORY.rst | 9 +++++++++ requests/__init__.py | 4 ++-- requests/models.py | 17 +++++++++++++++++ requests/sessions.py | 15 +++++---------- test_requests.py | 2 ++ 5 files changed, 35 insertions(+), 12 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 90ea6146..9625b95c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,15 @@ Release History --------------- +2.3.0 (YYYY-MM-DD) +++++++++++++++++++ + +**API Changes** + +- New ``Response`` property ``is_redirect``, which is true when the + library could have processed this response as a redirection (whether + or not it actually did). + 2.2.1 (2014-01-23) ++++++++++++++++++ diff --git a/requests/__init__.py b/requests/__init__.py index 2e9f3a0b..bba19002 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -42,8 +42,8 @@ is at . """ __title__ = 'requests' -__version__ = '2.2.1' -__build__ = 0x020201 +__version__ = '2.3.0' +__build__ = 0x020300 __author__ = 'Kenneth Reitz' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2014 Kenneth Reitz' diff --git a/requests/models.py b/requests/models.py index 25956be5..cd232e68 100644 --- a/requests/models.py +++ b/requests/models.py @@ -30,7 +30,17 @@ from .utils import ( from .compat import ( cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO, is_py2, chardet, json, builtin_str, basestring, IncompleteRead) +from .status_codes import codes +#: The set of HTTP status codes that indicate an automatically +#: processable redirect. +REDIRECT_STATI = ( + codes.moved, # 301 + codes.found, # 302 + codes.other, # 303 + codes.temporary_moved, # 307 +) +DEFAULT_REDIRECT_LIMIT = 30 CONTENT_CHUNK_SIZE = 10 * 1024 ITER_CHUNK_SIZE = 512 @@ -589,6 +599,13 @@ class Response(object): return False return True + @property + def is_redirect(self): + """True if this Response is a well-formed HTTP redirect that could have + been processed automatically (by :meth:`Session.resolve_redirects`). + """ + return ('location' in self.headers and self.status_code in REDIRECT_STATI) + @property def apparent_encoding(self): """The apparent encoding, provided by the chardet library""" diff --git a/requests/sessions.py b/requests/sessions.py index 2236e83f..a023e4ec 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -15,7 +15,7 @@ from datetime import datetime from .compat import cookielib, OrderedDict, urljoin, urlparse, builtin_str from .cookies import ( cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies) -from .models import Request, PreparedRequest +from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT from .hooks import default_hooks, dispatch_hook from .utils import to_key_val_list, default_headers, to_native_string from .exceptions import TooManyRedirects, InvalidSchema @@ -26,13 +26,9 @@ from .adapters import HTTPAdapter from .utils import requote_uri, get_environ_proxies, get_netrc_auth from .status_codes import codes -REDIRECT_STATI = ( - codes.moved, # 301 - codes.found, # 302 - codes.other, # 303 - codes.temporary_moved, # 307 -) -DEFAULT_REDIRECT_LIMIT = 30 + +# formerly defined here, reexposed here for backward compatibility +from .models import REDIRECT_STATI def merge_setting(request_setting, session_setting, dict_class=OrderedDict): @@ -89,8 +85,7 @@ class SessionRedirectMixin(object): i = 0 - # ((resp.status_code is codes.see_other)) - while ('location' in resp.headers and resp.status_code in REDIRECT_STATI): + while resp.is_redirect: prepared_request = req.copy() resp.content # Consume socket so it can be released diff --git a/test_requests.py b/test_requests.py index ee9c7b78..3d7cdaaf 100755 --- a/test_requests.py +++ b/test_requests.py @@ -115,6 +115,8 @@ class RequestsTestCase(unittest.TestCase): def test_HTTP_302_ALLOW_REDIRECT_GET(self): r = requests.get(httpbin('redirect', '1')) assert r.status_code == 200 + assert r.history[0].status_code == 302 + assert r.history[0].is_redirect # def test_HTTP_302_ALLOW_REDIRECT_POST(self): # r = requests.post(httpbin('status', '302'), data={'some': 'data'}) From d2f647cee45fd05cc1977cc3faf4b095b5047b29 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 14 Feb 2014 16:15:21 -0600 Subject: [PATCH 14/28] Do not set headers with None value - Regardless of whether they are on the session or not - Fixes #1920 --- requests/sessions.py | 2 ++ test_requests.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/requests/sessions.py b/requests/sessions.py index a023e4ec..c06fbcbe 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -59,6 +59,8 @@ def merge_setting(request_setting, session_setting, dict_class=OrderedDict): if v is None: del merged_setting[k] + merged_setting = dict((k, v) for (k, v) in merged_setting.items() if v is not None) + return merged_setting diff --git a/test_requests.py b/test_requests.py index 3d7cdaaf..998f17f3 100755 --- a/test_requests.py +++ b/test_requests.py @@ -211,6 +211,14 @@ class RequestsTestCase(unittest.TestCase): req_urls = [r.request.url for r in resp.history] assert urls == req_urls + def test_headers_on_session_with_None_are_not_sent(self): + """Do not send headers in Session.headers with None values.""" + ses = requests.Session() + ses.headers['Accept-Encoding'] = None + req = requests.Request('GET', 'http://httpbin.org/get') + prep = ses.prepare_request(req) + assert 'Accept-Encoding' not in prep.headers + def test_user_agent_transfers(self): heads = { From 81e88b70bd757f5ca5e9bc418a17369c13c6abf2 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Sun, 16 Feb 2014 19:00:22 +0000 Subject: [PATCH 15/28] The timeout is in seconds. --- requests/api.py | 2 +- requests/sessions.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/api.py b/requests/api.py index baf43dd6..01d853d5 100644 --- a/requests/api.py +++ b/requests/api.py @@ -26,7 +26,7 @@ def request(method, url, **kwargs): :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of 'name': file-like-objects (or {'name': ('filename', fileobj)}) for multipart encoding upload. :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. - :param timeout: (optional) Float describing the timeout of the request. + :param timeout: (optional) Float describing the timeout of the request in seconds. :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided. diff --git a/requests/sessions.py b/requests/sessions.py index a023e4ec..b2dc1a99 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -328,7 +328,7 @@ class Session(SessionRedirectMixin): :param auth: (optional) Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth. :param timeout: (optional) Float describing the timeout of the - request. + request in seconds. :param allow_redirects: (optional) Boolean. Set to True by default. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. From 9b20cadaeb8103ab37c4f02f5cfcd6ca7ca7ec22 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Fri, 28 Feb 2014 08:27:50 -0600 Subject: [PATCH 16/28] Charade -> Chardet and Add cacert.pem license - Charade is gone, long live Chardet. - cacert.pem is now taken wholesale from Mozilla so we need to display that itis licensed under the MPL2.0 --- NOTICE | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/NOTICE b/NOTICE index 4d69475c..76641d57 100644 --- a/NOTICE +++ b/NOTICE @@ -24,8 +24,8 @@ FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TOR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -Charade License -================ +Chardet License +=============== This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public @@ -46,18 +46,14 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA CA Bundle License ================= -This library is free software; you can redistribute it and/or -modify it under the terms of the GNU Lesser General Public -License as published by the Free Software Foundation; either -version 2.1 of the License, or (at your option) any later version. - -This library is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -Lesser General Public License for more details. - -You should have received a copy of the GNU Lesser General Public -License along with this library; if not, write to the Free Software -Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA -02110-1301 +This Source Code Form is subject to the terms of the Mozilla Public +License, v. 2.0. If a copy of the MPL was not distributed with this +file, You can obtain one at http://mozilla.org/MPL/2.0/. +Issuer: CN=GTE CyberTrust Global Root O=GTE Corporation OU=GTE CyberTrust Solutions, Inc. +Subject: CN=GTE CyberTrust Global Root O=GTE Corporation OU=GTE CyberTrust Solutions, Inc. +Label: "GTE CyberTrust Global Root" +Serial: 421 +MD5 Fingerprint: ca:3d:d3:68:f1:03:5c:d0:32:fa:b8:2b:59:e8:5a:db +SHA1 Fingerprint: 97:81:79:50:d8:1c:96:70:cc:34:d8:09:cf:79:44:31:36:7e:f4:74 +SHA256 Fingerprint: a5:31:25:18:8d:21:10:aa:96:4b:02:c7:b7:c6:da:32:03:17:08:94:e5:fb:71:ff:fb:66:67:d5:e6:81:0a:36 From 930f03c8649613da9efb7915e17428dd1e8526af Mon Sep 17 00:00:00 2001 From: cjstapleton Date: Fri, 28 Feb 2014 10:08:57 -0600 Subject: [PATCH 17/28] Add timeout to stream with testing Fixes Issue #1803 --- requests/adapters.py | 5 +---- test_requests.py | 9 +++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index ca462232..28bea07c 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -310,10 +310,7 @@ class HTTPAdapter(BaseAdapter): chunked = not (request.body is None or 'Content-Length' in request.headers) - if stream: - timeout = TimeoutSauce(connect=timeout) - else: - timeout = TimeoutSauce(connect=timeout, read=timeout) + timeout = TimeoutSauce(connect=timeout, read=timeout) try: if not chunked: diff --git a/test_requests.py b/test_requests.py index 3d7cdaaf..0fe849d4 100755 --- a/test_requests.py +++ b/test_requests.py @@ -1170,6 +1170,15 @@ class TestMorselToCookieMaxAge(unittest.TestCase): with pytest.raises(TypeError): morsel_to_cookie(morsel) +class TestTimeout: + def test_stream_timeout(self): + try: + r = requests.get('https://httpbin.org/delay/10', timeout=5.0) + except requests.exceptions.Timeout as e: + assert 'Read timed out' in e.args[0].args[0] + if __name__ == '__main__': unittest.main() + + From 5f404a0592bba18c5eae7b2aead033f3d0cb27ff Mon Sep 17 00:00:00 2001 From: cjstapleton Date: Fri, 28 Feb 2014 19:06:59 -0600 Subject: [PATCH 18/28] Fix styling issues with add timeout to stream with testing --- test_requests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test_requests.py b/test_requests.py index 0fe849d4..a6b6f89c 100755 --- a/test_requests.py +++ b/test_requests.py @@ -1170,6 +1170,7 @@ class TestMorselToCookieMaxAge(unittest.TestCase): with pytest.raises(TypeError): morsel_to_cookie(morsel) + class TestTimeout: def test_stream_timeout(self): try: @@ -1180,5 +1181,3 @@ class TestTimeout: if __name__ == '__main__': unittest.main() - - From 64f0b3c81ecbe6b4dff1393ad0409da64e1e8cee Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 1 Mar 2014 11:15:14 -0600 Subject: [PATCH 19/28] Remove unnecessary bits from cacert notice section --- NOTICE | 8 -------- 1 file changed, 8 deletions(-) diff --git a/NOTICE b/NOTICE index 76641d57..223101a0 100644 --- a/NOTICE +++ b/NOTICE @@ -49,11 +49,3 @@ CA Bundle License This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -Issuer: CN=GTE CyberTrust Global Root O=GTE Corporation OU=GTE CyberTrust Solutions, Inc. -Subject: CN=GTE CyberTrust Global Root O=GTE Corporation OU=GTE CyberTrust Solutions, Inc. -Label: "GTE CyberTrust Global Root" -Serial: 421 -MD5 Fingerprint: ca:3d:d3:68:f1:03:5c:d0:32:fa:b8:2b:59:e8:5a:db -SHA1 Fingerprint: 97:81:79:50:d8:1c:96:70:cc:34:d8:09:cf:79:44:31:36:7e:f4:74 -SHA256 Fingerprint: a5:31:25:18:8d:21:10:aa:96:4b:02:c7:b7:c6:da:32:03:17:08:94:e5:fb:71:ff:fb:66:67:d5:e6:81:0a:36 From 03f444e601e6608f68f3a09d708af93e4512560a Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 3 Mar 2014 20:23:59 +0000 Subject: [PATCH 20/28] Timeout documentation changes. --- HISTORY.rst | 2 ++ docs/api.rst | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 9625b95c..19c2ff1b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,8 @@ Release History - New ``Response`` property ``is_redirect``, which is true when the library could have processed this response as a redirection (whether or not it actually did). +- The ``timeout`` parameter now affects requests with both ``stream=True`` and + ``stream=False`` equally. 2.2.1 (2014-01-23) ++++++++++++++++++ diff --git a/docs/api.rst b/docs/api.rst index 77f1f02d..86061be9 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -249,7 +249,9 @@ Behavioral Changes * Timeouts behave slightly differently. On streaming requests, the timeout only applies to the connection attempt. On regular requests, the timeout - is applied to the connection process and downloading the full body. + is applied to the connection process and on to all attempts to read data from + the underlying socket. It does *not* apply to the total download time for the + request. :: From a59fac5e4ba290164cd0e108769bcab523f1bfb2 Mon Sep 17 00:00:00 2001 From: schlamar Date: Wed, 12 Mar 2014 12:10:53 +0100 Subject: [PATCH 21/28] Removed compress from accepted encodings. --- requests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/utils.py b/requests/utils.py index 7b7ff0a7..4d648bc5 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -548,7 +548,7 @@ def default_user_agent(name="python-requests"): def default_headers(): return CaseInsensitiveDict({ 'User-Agent': default_user_agent(), - 'Accept-Encoding': ', '.join(('gzip', 'deflate', 'compress')), + 'Accept-Encoding': ', '.join(('gzip', 'deflate')), 'Accept': '*/*' }) From ee7fe02953d864021298ed8d3e3e5f6aff1f6731 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 12 Mar 2014 18:34:43 +0000 Subject: [PATCH 22/28] Ensure that .raw() is present after unpickling. Some people will assume that .raw() is present, and they shouldn't get AttributeErrors when they make that assumption on a pickled Response. However, @kennethreitz has asked that we not be too dependent on urllib3. For that reason, set to None. --- requests/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/requests/models.py b/requests/models.py index cd232e68..e2fa09f8 100644 --- a/requests/models.py +++ b/requests/models.py @@ -575,6 +575,7 @@ class Response(object): # pickled objects do not have .raw setattr(self, '_content_consumed', True) + setattr(self, 'raw', None) def __repr__(self): return '' % (self.status_code) From 6d7e8a97bbefa287366bc5d0b0b8f789532e853a Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 12 Mar 2014 19:20:12 +0000 Subject: [PATCH 23/28] Split get_environ_proxies into two methods. This makes it possible to get at the no_proxy logic separately. --- requests/utils.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 7b7ff0a7..1095b3e7 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -466,9 +466,10 @@ def is_valid_cidr(string_network): return True -def get_environ_proxies(url): - """Return a dict of environment proxies.""" - +def should_bypass_proxies(url): + """ + Returns whether we should bypass proxies or not. + """ get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper()) # First check whether no_proxy is defined. If it is, check that the URL @@ -486,13 +487,13 @@ def get_environ_proxies(url): for proxy_ip in no_proxy: if is_valid_cidr(proxy_ip): if address_in_network(ip, proxy_ip): - return {} + return True else: for host in no_proxy: if netloc.endswith(host) or netloc.split(':')[0].endswith(host): # The URL does match something in no_proxy, so we don't want # to apply the proxies on this URL. - return {} + return True # If the system proxy settings indicate that this URL should be bypassed, # don't proxy. @@ -506,12 +507,16 @@ def get_environ_proxies(url): bypass = False if bypass: - return {} + return True - # If we get here, we either didn't have no_proxy set or we're not going - # anywhere that no_proxy applies to, and the system settings don't require - # bypassing the proxy for the current URL. - return getproxies() + return False + +def get_environ_proxies(url): + """Return a dict of environment proxies.""" + if should_bypass_proxies(url): + return {} + else: + return getproxies() def default_user_agent(name="python-requests"): From 97cf16e958a948ecf30c3019ae94f2e7ec7dcb7f Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 12 Mar 2014 19:22:11 +0000 Subject: [PATCH 24/28] Move auth rebuild to its own method. --- requests/sessions.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index 425db22c..4c24984e 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -154,19 +154,7 @@ class SessionRedirectMixin(object): prepared_request._cookies.update(self.cookies) prepared_request.prepare_cookies(prepared_request._cookies) - if 'Authorization' in headers: - # If we get redirected to a new host, we should strip out any - # authentication headers. - original_parsed = urlparse(resp.request.url) - redirect_parsed = urlparse(url) - - if (original_parsed.hostname != redirect_parsed.hostname): - del headers['Authorization'] - - # .netrc might have more auth for us. - new_auth = get_netrc_auth(url) if self.trust_env else None - if new_auth is not None: - prepared_request.prepare_auth(new_auth) + self.rebuild_auth(prepared_request, resp) resp = self.send( prepared_request, @@ -183,6 +171,31 @@ class SessionRedirectMixin(object): i += 1 yield resp + def rebuild_auth(self, prepared_request, response): + """ + When being redirected we may want to strip authentication from the + request to avoid leaking credentials. This method intelligently removes + and reapplies authentication where possible to avoid credential loss. + """ + headers = prepared_request.headers + url = prepared_request.url + + if 'Authorization' in headers: + # If we get redirected to a new host, we should strip out any + # authentication headers. + original_parsed = urlparse(response.request.url) + redirect_parsed = urlparse(url) + + if (original_parsed.hostname != redirect_parsed.hostname): + del headers['Authorization'] + + # .netrc might have more auth for us on our new host. + new_auth = get_netrc_auth(url) if self.trust_env else None + if new_auth is not None: + prepared_request.prepare_auth(new_auth) + + return + class Session(SessionRedirectMixin): """A Requests session. From 4d8cb3244e8e4f84b250c10a48e025f9a8bf6137 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 12 Mar 2014 19:22:40 +0000 Subject: [PATCH 25/28] Add method for rebuilding proxy configuration. This includes auth. --- requests/sessions.py | 52 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/requests/sessions.py b/requests/sessions.py index 4c24984e..db2fca39 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -12,6 +12,7 @@ import os from collections import Mapping from datetime import datetime +from .auth import _basic_auth_str from .compat import cookielib, OrderedDict, urljoin, urlparse, builtin_str from .cookies import ( cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies) @@ -23,7 +24,10 @@ from .structures import CaseInsensitiveDict from .adapters import HTTPAdapter -from .utils import requote_uri, get_environ_proxies, get_netrc_auth +from .utils import ( + requote_uri, get_environ_proxies, get_netrc_auth, should_bypass_proxies, + get_auth_from_url +) from .status_codes import codes @@ -154,6 +158,8 @@ class SessionRedirectMixin(object): prepared_request._cookies.update(self.cookies) prepared_request.prepare_cookies(prepared_request._cookies) + # Rebuild auth and proxy information. + proxies = self.rebuild_proxies(prepared_request, proxies) self.rebuild_auth(prepared_request, resp) resp = self.send( @@ -196,6 +202,50 @@ class SessionRedirectMixin(object): return + def rebuild_proxies(self, prepared_request, proxies): + """ + This method re-evaluates the proxy configuration by considering the + environment variables. If we are redirected to a URL covered by + NO_PROXY, we strip the proxy configuration. Otherwise, we set missing + proxy keys for this URL (in case they were stripped by a previous + redirect). + + This method also replaces the Proxy-Authorization header where + necessary. + """ + headers = prepared_request.headers + url = prepared_request.url + new_proxies = {} + + # Consider proxies. First evaluate the new proxy config. If we are + # being redirected to a host on the NO_PROXY list then we want to + # remove the proxy dictionary entirely. Otherwise, if there's a relevant + # environment proxy, set it if we don't already have a proxy to go to. + if not should_bypass_proxies(url): + environ_proxies = get_environ_proxies(url) + scheme = urlparse(url).scheme + + try: + new_proxies.setdefault(scheme, environ_proxies[scheme]) + except KeyError: + pass + + # If there's a proxy-authorization header present, remove it, then add + # a new one (potentially re-adding the one we just removed). + if 'Proxy-Authorization' in headers: + del headers['Proxy-Authorization'] + + try: + username, password = get_auth_from_url(new_proxies[scheme]) + if username and password: + headers['Proxy-Authorization'] = _basic_auth_str( + username, password + ) + except KeyError: + pass + + return new_proxies + class Session(SessionRedirectMixin): """A Requests session. From 4f6dca42ea0fb3d1c4706e63e594e43f7a3237f7 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 12 Mar 2014 21:21:14 +0000 Subject: [PATCH 26/28] Remove some extraneous comments to please KR. --- requests/sessions.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index db2fca39..65902d87 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -217,10 +217,6 @@ class SessionRedirectMixin(object): url = prepared_request.url new_proxies = {} - # Consider proxies. First evaluate the new proxy config. If we are - # being redirected to a host on the NO_PROXY list then we want to - # remove the proxy dictionary entirely. Otherwise, if there's a relevant - # environment proxy, set it if we don't already have a proxy to go to. if not should_bypass_proxies(url): environ_proxies = get_environ_proxies(url) scheme = urlparse(url).scheme @@ -230,8 +226,6 @@ class SessionRedirectMixin(object): except KeyError: pass - # If there's a proxy-authorization header present, remove it, then add - # a new one (potentially re-adding the one we just removed). if 'Proxy-Authorization' in headers: del headers['Proxy-Authorization'] From 724038e4b5e94addb0bf6c767b7f5578b172c659 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 12 Mar 2014 21:53:07 +0000 Subject: [PATCH 27/28] Test stripping standard auth. --- test_requests.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test_requests.py b/test_requests.py index 17de8491..1bebb1ad 100755 --- a/test_requests.py +++ b/test_requests.py @@ -17,6 +17,7 @@ from requests.compat import ( Morsel, cookielib, getproxies, str, urljoin, urlparse) from requests.cookies import cookiejar_from_dict, morsel_to_cookie from requests.exceptions import InvalidURL, MissingSchema +from requests.models import PreparedRequest, Response from requests.structures import CaseInsensitiveDict try: @@ -865,6 +866,22 @@ class RequestsTestCase(unittest.TestCase): preq = req.prepare() assert test_url == preq.url + def test_auth_is_stripped_on_redirect_off_host(self): + r = requests.get( + httpbin('redirect-to'), + params={'url': 'http://www.google.co.uk'}, + auth=('user', 'pass'), + ) + assert r.history[0].request.headers['Authorization'] + assert not r.request.headers.get('Authorization', '') + + def test_auth_is_retained_for_redirect_on_host(self): + r = requests.get(httpbin('redirect/1'), auth=('user', 'pass')) + h1 = r.history[0].request.headers['Authorization'] + h2 = r.request.headers['Authorization'] + + assert h1 == h2 + class TestContentEncodingDetection(unittest.TestCase): From 90f73378582e4e2cbc75a189a2cfa7826824f29e Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Sat, 22 Mar 2014 21:11:33 +0000 Subject: [PATCH 28/28] Style changes thanks to @sigmavirus24. --- requests/sessions.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index 65902d87..79ea7773 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -221,22 +221,21 @@ class SessionRedirectMixin(object): environ_proxies = get_environ_proxies(url) scheme = urlparse(url).scheme - try: + proxy = environ_proxies.get(scheme) + + if proxy: new_proxies.setdefault(scheme, environ_proxies[scheme]) - except KeyError: - pass if 'Proxy-Authorization' in headers: del headers['Proxy-Authorization'] try: username, password = get_auth_from_url(new_proxies[scheme]) - if username and password: - headers['Proxy-Authorization'] = _basic_auth_str( - username, password - ) except KeyError: - pass + username, password = None, None + + if username and password: + headers['Proxy-Authorization'] = _basic_auth_str(username, password) return new_proxies