From 2c153c155659e5be198fa1959907d80b6c5de67c Mon Sep 17 00:00:00 2001 From: Jason Emerick Date: Mon, 19 Dec 2011 11:45:32 -0500 Subject: [PATCH 01/14] Add Twitter Streaming API example to Advanced Usage docs --- docs/user/advanced.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 099dabd7..4b3430f2 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -243,6 +243,27 @@ Then, we can make a request using our Pizza Auth:: +Streaming Requests +------------------ + +With ``requests.Response.iter_lines()`` you can easily iterate over streaming +APIs such as the `Twitter Streaming API `_. + +To use the Twitter Streaming API to track the keyword "requests": + +:: + + import requests + import json + + r = requests.post('https://stream.twitter.com/1/statuses/filter.json', + data={'track': 'requests'}, auth=('username', 'password')) + + for line in r.iter_lines(): + if line: # filter out keep-alive new lines + print json.loads(line) + + Verbose Logging --------------- From f80fbad4ff95152021f68703149c371f5b44c7cf Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 19 Dec 2011 12:25:59 -0500 Subject: [PATCH 02/14] Update AUTHORS.rst --- AUTHORS.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 023deb52..bf61d06c 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -65,4 +65,5 @@ Patches and Suggestions - Idan Gazit - Ed Summers - Chris Van Horne -- Christopher Davis \ No newline at end of file +- Christopher Davis +- Jason Emerick \ No newline at end of file From f214b91d770ef194f9fad7041a1356ff27097ae5 Mon Sep 17 00:00:00 2001 From: Ori Livneh Date: Tue, 20 Dec 2011 02:01:37 -0500 Subject: [PATCH 03/14] fix bug that causes iter_lines to truncate content Currently, if the response does not terminate with a newline, iter_lines truncates the trailing remainder. --- requests/models.py | 7 ++++++- test_requests.py | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index e84c8f1a..88a002a0 100644 --- a/requests/models.py +++ b/requests/models.py @@ -597,7 +597,7 @@ class Response(object): def generate(): chunk = [] - while 1: + while True: c = self.raw.read(1) if not c: break @@ -608,6 +608,11 @@ class Response(object): else: chunk.append(c) + # Yield the remainder, in case the response + # did not terminate with a newline + if chunk: + yield ''.join(chunk) + self._content_consumed = True gen = generate() diff --git a/test_requests.py b/test_requests.py index 829d2a94..ebcb596d 100755 --- a/test_requests.py +++ b/test_requests.py @@ -3,6 +3,7 @@ from __future__ import with_statement +import StringIO import time import os import unittest @@ -603,5 +604,21 @@ class RequestsTestSuite(unittest.TestCase): self.assertEqual(i, len_lines) + # Test 'dangling' fragment in responses that do not terminate in + # a newline. + quote = ( + '''Why will he not upon our fair request\n''' + '''Untent his person and share the air with us?''' + ) + + # Make a request and monkey-patch its contents + r = requests.get(httpbin('get')) + r.raw = StringIO.StringIO(quote) + + # Make sure iter_lines doesn't chop the trailing bit + lines = '\n'.join(r.iter_lines()) + self.assertEqual(lines, quote) + + if __name__ == '__main__': unittest.main() From 75dc012f41d26c2eb8348cae319b26721ec4f410 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 22 Dec 2011 11:54:07 -0500 Subject: [PATCH 04/14] AUTHORS += 'Ori Livneh' #313 --- AUTHORS.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 023deb52..0715f2d8 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -65,4 +65,5 @@ Patches and Suggestions - Idan Gazit - Ed Summers - Chris Van Horne -- Christopher Davis \ No newline at end of file +- Christopher Davis +- Ori Livneh From f5e421e880761791bbec40556aa9abf15bf7f4f8 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 22 Dec 2011 11:55:55 -0500 Subject: [PATCH 05/14] 1 instead of True #313 --- requests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 88a002a0..4bef293d 100644 --- a/requests/models.py +++ b/requests/models.py @@ -597,7 +597,7 @@ class Response(object): def generate(): chunk = [] - while True: + while 1: c = self.raw.read(1) if not c: break From 12f3c65e5a426ce32808cc456387ca7d7e7c5126 Mon Sep 17 00:00:00 2001 From: Ori Livneh Date: Thu, 22 Dec 2011 16:52:29 -0500 Subject: [PATCH 06/14] Make safe_mode play nice with timeout (fixes #312) --- requests/models.py | 8 +++++--- test_requests.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index 4bef293d..f9a9626b 100644 --- a/requests/models.py +++ b/requests/models.py @@ -438,6 +438,10 @@ class Request(object): # Attach Cookie header to request. self.headers['Cookie'] = cookie_header + # If the request fails but exceptions are suppressed, + # we'll build a blank response. + r = None + try: # Send the request. r = conn.urlopen( @@ -458,8 +462,6 @@ class Request(object): except MaxRetryError, e: if not self.config.get('safe_mode', False): raise ConnectionError(e) - else: - r = None except (_SSLError, _HTTPError), e: if not self.config.get('safe_mode', False): @@ -597,7 +599,7 @@ class Response(object): def generate(): chunk = [] - while 1: + while True: c = self.raw.read(1) if not c: break diff --git a/test_requests.py b/test_requests.py index ebcb596d..e547aa64 100755 --- a/test_requests.py +++ b/test_requests.py @@ -619,6 +619,17 @@ class RequestsTestSuite(unittest.TestCase): lines = '\n'.join(r.iter_lines()) self.assertEqual(lines, quote) + def test_timeout(self): + + # When not in safe mode, should raise Timeout exception + with self.assertRaises(requests.exceptions.Timeout): + r = requests.get(httpbin('stream', '1000'), timeout=0.0001) + + # In safe mode, should return a blank response + r = requests.get(httpbin('stream', '1000'), timeout=0.0001, + config=dict(safe_mode=True)) + self.assertIsNone(r.content) + if __name__ == '__main__': unittest.main() From 91766548d10cd5ebf46aba376f9e000263285875 Mon Sep 17 00:00:00 2001 From: Ori Livneh Date: Thu, 22 Dec 2011 17:36:09 -0500 Subject: [PATCH 07/14] Revert "Make safe_mode play nice with timeout (fixes #312)" This reverts commit 805c4db55413c96c5d6e0f95560119c000f660c9. --- requests/models.py | 8 +++----- test_requests.py | 11 ----------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/requests/models.py b/requests/models.py index f9a9626b..4bef293d 100644 --- a/requests/models.py +++ b/requests/models.py @@ -438,10 +438,6 @@ class Request(object): # Attach Cookie header to request. self.headers['Cookie'] = cookie_header - # If the request fails but exceptions are suppressed, - # we'll build a blank response. - r = None - try: # Send the request. r = conn.urlopen( @@ -462,6 +458,8 @@ class Request(object): except MaxRetryError, e: if not self.config.get('safe_mode', False): raise ConnectionError(e) + else: + r = None except (_SSLError, _HTTPError), e: if not self.config.get('safe_mode', False): @@ -599,7 +597,7 @@ class Response(object): def generate(): chunk = [] - while True: + while 1: c = self.raw.read(1) if not c: break diff --git a/test_requests.py b/test_requests.py index e547aa64..ebcb596d 100755 --- a/test_requests.py +++ b/test_requests.py @@ -619,17 +619,6 @@ class RequestsTestSuite(unittest.TestCase): lines = '\n'.join(r.iter_lines()) self.assertEqual(lines, quote) - def test_timeout(self): - - # When not in safe mode, should raise Timeout exception - with self.assertRaises(requests.exceptions.Timeout): - r = requests.get(httpbin('stream', '1000'), timeout=0.0001) - - # In safe mode, should return a blank response - r = requests.get(httpbin('stream', '1000'), timeout=0.0001, - config=dict(safe_mode=True)) - self.assertIsNone(r.content) - if __name__ == '__main__': unittest.main() From 2c0ff95d4d8daae7c0624b3a96584d722ebf6c38 Mon Sep 17 00:00:00 2001 From: Ori Livneh Date: Thu, 22 Dec 2011 17:50:43 -0500 Subject: [PATCH 08/14] Revert "Revert "Make safe_mode play nice with timeout (fixes #312)"" This reverts commit 91766548d10cd5ebf46aba376f9e000263285875. --- requests/models.py | 8 +++++--- test_requests.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index 4bef293d..f9a9626b 100644 --- a/requests/models.py +++ b/requests/models.py @@ -438,6 +438,10 @@ class Request(object): # Attach Cookie header to request. self.headers['Cookie'] = cookie_header + # If the request fails but exceptions are suppressed, + # we'll build a blank response. + r = None + try: # Send the request. r = conn.urlopen( @@ -458,8 +462,6 @@ class Request(object): except MaxRetryError, e: if not self.config.get('safe_mode', False): raise ConnectionError(e) - else: - r = None except (_SSLError, _HTTPError), e: if not self.config.get('safe_mode', False): @@ -597,7 +599,7 @@ class Response(object): def generate(): chunk = [] - while 1: + while True: c = self.raw.read(1) if not c: break diff --git a/test_requests.py b/test_requests.py index ebcb596d..e547aa64 100755 --- a/test_requests.py +++ b/test_requests.py @@ -619,6 +619,17 @@ class RequestsTestSuite(unittest.TestCase): lines = '\n'.join(r.iter_lines()) self.assertEqual(lines, quote) + def test_timeout(self): + + # When not in safe mode, should raise Timeout exception + with self.assertRaises(requests.exceptions.Timeout): + r = requests.get(httpbin('stream', '1000'), timeout=0.0001) + + # In safe mode, should return a blank response + r = requests.get(httpbin('stream', '1000'), timeout=0.0001, + config=dict(safe_mode=True)) + self.assertIsNone(r.content) + if __name__ == '__main__': unittest.main() From a7223f86a976410bb904fb2e983440c94175aa75 Mon Sep 17 00:00:00 2001 From: Ori Livneh Date: Thu, 22 Dec 2011 17:51:22 -0500 Subject: [PATCH 09/14] Undo stupid change to while loop --- requests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index f9a9626b..09ea87d9 100644 --- a/requests/models.py +++ b/requests/models.py @@ -599,7 +599,7 @@ class Response(object): def generate(): chunk = [] - while True: + while 1: c = self.raw.read(1) if not c: break From 716d29c49c346106b25fadd1d15d6bd79cd6847a Mon Sep 17 00:00:00 2001 From: Ori Livneh Date: Thu, 22 Dec 2011 21:10:43 -0500 Subject: [PATCH 10/14] fix iteration on null responses in safe_mode --- requests/models.py | 41 ++++++++++++++++++++++------------------- test_requests.py | 11 +++++++++++ 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/requests/models.py b/requests/models.py index 09ea87d9..a51ee700 100644 --- a/requests/models.py +++ b/requests/models.py @@ -559,11 +559,13 @@ class Response(object): ) def generate(): - while 1: - chunk = self.raw.read(chunk_size) - if not chunk: - break - yield chunk + # self.raw can be None if we're in safe_mode and the request failed + if self.raw is not None: + while 1: + chunk = self.raw.read(chunk_size) + if not chunk: + break + yield chunk self._content_consumed = True gen = generate() @@ -597,23 +599,24 @@ class Response(object): ) def generate(): - chunk = [] + if self.raw is not None: + chunk = [] - while 1: - c = self.raw.read(1) - if not c: - break + while 1: + c = self.raw.read(1) + if not c: + break - if c in newlines: + if c in newlines: + yield ''.join(chunk) + chunk = [] + else: + chunk.append(c) + + # Yield the remainder, in case the response + # did not terminate with a newline + if chunk: yield ''.join(chunk) - chunk = [] - else: - chunk.append(c) - - # Yield the remainder, in case the response - # did not terminate with a newline - if chunk: - yield ''.join(chunk) self._content_consumed = True diff --git a/test_requests.py b/test_requests.py index e547aa64..2543103f 100755 --- a/test_requests.py +++ b/test_requests.py @@ -619,6 +619,17 @@ class RequestsTestSuite(unittest.TestCase): lines = '\n'.join(r.iter_lines()) self.assertEqual(lines, quote) + def test_null_response(self): + + # Safe mode creates empty responses for failed requests. + + # Iterating on these responses should produce empty sequences + r = requests.get('http://_/', config=dict(safe_mode=True)) + self.assertEquals(list(r.iter_lines()), []) + + r = requests.get('http://_/', config=dict(safe_mode=True)) + self.assertEquals(list(r.iter_content()), []) + def test_timeout(self): # When not in safe mode, should raise Timeout exception From 3483bae24f0fc5f18f7466fffa83584068474a59 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 22 Dec 2011 23:09:27 -0500 Subject: [PATCH 11/14] Update docs/index.rst --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 3ae0a728..677d615f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -38,7 +38,7 @@ Testimonials `Twitter, Inc `_, `Readability `_, and Federal US Institutions -use Requests internally. It has been installed over 45,000 times from PyPI. +use Requests internally. It has been installed over 60,000 times from PyPI. **Armin Ronacher** Requests is the perfect example how beautiful an API can be with the From e1615e0f1e342d89d09df8289c0b32d3b0e9fd6a Mon Sep 17 00:00:00 2001 From: Ori Livneh Date: Fri, 23 Dec 2011 16:14:46 -0500 Subject: [PATCH 12/14] Make safe_mode handle exceptions more consistently --- requests/models.py | 71 ++++++++++++++++++++++++++-------------------- test_requests.py | 14 +++++---- 2 files changed, 48 insertions(+), 37 deletions(-) diff --git a/requests/models.py b/requests/models.py index a51ee700..1611bd07 100644 --- a/requests/models.py +++ b/requests/models.py @@ -18,13 +18,15 @@ from .structures import CaseInsensitiveDict from .status_codes import codes from .packages import oreos from .auth import HTTPBasicAuth, HTTPProxyAuth +from .packages.urllib3.response import HTTPResponse from .packages.urllib3.exceptions import MaxRetryError from .packages.urllib3.exceptions import SSLError as _SSLError from .packages.urllib3.exceptions import HTTPError as _HTTPError from .packages.urllib3 import connectionpool, poolmanager from .packages.urllib3.filepost import encode_multipart_formdata from .exceptions import ( - Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError) + ConnectionError, HTTPError, RequestException, Timeout, TooManyRedirects, + URLRequired) from .utils import ( get_encoding_from_headers, stream_decode_response_unicode, decode_gzip, stream_decode_gzip, guess_filename, requote_path) @@ -171,6 +173,9 @@ class Request(object): # Save cookies in Response. response.cookies = cookies + # No exceptions were harmed in the making of this request. + response.error = getattr(resp, 'error', None) + # Save original response for later. response.raw = resp @@ -438,35 +443,41 @@ class Request(object): # Attach Cookie header to request. self.headers['Cookie'] = cookie_header - # If the request fails but exceptions are suppressed, - # we'll build a blank response. - r = None - try: - # Send the request. - r = conn.urlopen( - method=self.method, - url=self.path_url, - body=body, - headers=self.headers, - redirect=False, - assert_same_host=False, - preload_content=False, - decode_content=False, - retries=self.config.get('max_retries', 0), - timeout=self.timeout, - ) - self.sent = True + # The inner try .. except re-raises certain exceptions as + # internal exception types; the outer suppresses exceptions + # when safe mode is set. + try: + # Send the request. + r = conn.urlopen( + method=self.method, + url=self.path_url, + body=body, + headers=self.headers, + redirect=False, + assert_same_host=False, + preload_content=False, + decode_content=False, + retries=self.config.get('max_retries', 0), + timeout=self.timeout, + ) + self.sent = True - - except MaxRetryError, e: - if not self.config.get('safe_mode', False): + except MaxRetryError, e: raise ConnectionError(e) - except (_SSLError, _HTTPError), e: - if not self.config.get('safe_mode', False): + except (_SSLError, _HTTPError), e: raise Timeout('Request timed out.') + except RequestException, e: + if self.config.get('safe_mode', False): + # In safe mode, catch the exception and attach it to + # a blank urllib3.HTTPResponse object. + r = HTTPResponse() + r.error = e + else: + raise + self._build_response(r) # Response manipulation hook. @@ -559,13 +570,11 @@ class Response(object): ) def generate(): - # self.raw can be None if we're in safe_mode and the request failed - if self.raw is not None: - while 1: - chunk = self.raw.read(chunk_size) - if not chunk: - break - yield chunk + while 1: + chunk = self.raw.read(chunk_size) + if not chunk: + break + yield chunk self._content_consumed = True gen = generate() diff --git a/test_requests.py b/test_requests.py index 2543103f..b3e55c8c 100755 --- a/test_requests.py +++ b/test_requests.py @@ -619,18 +619,19 @@ class RequestsTestSuite(unittest.TestCase): lines = '\n'.join(r.iter_lines()) self.assertEqual(lines, quote) - def test_null_response(self): + def test_safe_mode(self): + + safe = requests.session(config=dict(safe_mode=True)) # Safe mode creates empty responses for failed requests. - # Iterating on these responses should produce empty sequences - r = requests.get('http://_/', config=dict(safe_mode=True)) + r = safe.get('http://_/') self.assertEquals(list(r.iter_lines()), []) + self.assertIsInstance(r.error, requests.exceptions.ConnectionError) - r = requests.get('http://_/', config=dict(safe_mode=True)) + r = safe.get('http://_/') self.assertEquals(list(r.iter_content()), []) - - def test_timeout(self): + self.assertIsInstance(r.error, requests.exceptions.ConnectionError) # When not in safe mode, should raise Timeout exception with self.assertRaises(requests.exceptions.Timeout): @@ -640,6 +641,7 @@ class RequestsTestSuite(unittest.TestCase): r = requests.get(httpbin('stream', '1000'), timeout=0.0001, config=dict(safe_mode=True)) self.assertIsNone(r.content) + self.assertIsInstance(r.error, requests.exceptions.Timeout) if __name__ == '__main__': From 4eb2759400b6bc8a6289a4fd37779c21b6894b3c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 24 Dec 2011 03:06:13 -0500 Subject: [PATCH 13/14] force safe mode for async --- requests/async.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/requests/async.py b/requests/async.py index c91025fc..6f8de380 100644 --- a/requests/async.py +++ b/requests/async.py @@ -36,6 +36,11 @@ def patched(f): kwargs['return_response'] = False kwargs['prefetch'] = True + config = kwargs.get('config', {}) + config.update(safe_mode=True) + + kwargs['config'] = config + return f(*args, **kwargs) return wrapped From 0bfc029b5e549cb41b7c4999b96f3dfe892baf84 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 24 Dec 2011 03:16:40 -0500 Subject: [PATCH 14/14] v0.8.7 --- HISTORY.rst | 9 ++++++++- requests/__init__.py | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d17355ec..7b634a36 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,13 +1,20 @@ History ------- +0.8.7 (2011-12-24) +++++++++++++++++++ + +* iter_lines last-line truncation fix +* Force safe_mode for async requests +* Handle safe_mode exceptions more consistently +* Fix iteration on null responses in safe_mode + 0.8.6 (2011-12-18) ++++++++++++++++++ * Socket timeout fixes. * Proxy Authorization support. - 0.8.5 (2011-12-14) ++++++++++++++++++ diff --git a/requests/__init__.py b/requests/__init__.py index ee3173e1..847f0464 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -15,8 +15,8 @@ requests """ __title__ = 'requests' -__version__ = '0.8.6' -__build__ = 0x000806 +__version__ = '0.8.7' +__build__ = 0x000807 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz'