diff --git a/AUTHORS.rst b/AUTHORS.rst index 023deb52..b9c3a643 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -65,4 +65,6 @@ Patches and Suggestions - Idan Gazit - Ed Summers - Chris Van Horne -- Christopher Davis \ No newline at end of file +- Christopher Davis +- Ori Livneh +- Jason Emerick diff --git a/HISTORY.rst b/HISTORY.rst index d17355ec..7b634a36 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,13 +1,20 @@ History ------- +0.8.7 (2011-12-24) +++++++++++++++++++ + +* iter_lines last-line truncation fix +* Force safe_mode for async requests +* Handle safe_mode exceptions more consistently +* Fix iteration on null responses in safe_mode + 0.8.6 (2011-12-18) ++++++++++++++++++ * Socket timeout fixes. * Proxy Authorization support. - 0.8.5 (2011-12-14) ++++++++++++++++++ diff --git a/docs/index.rst b/docs/index.rst index 3ae0a728..677d615f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -38,7 +38,7 @@ Testimonials `Twitter, Inc `_, `Readability `_, and Federal US Institutions -use Requests internally. It has been installed over 45,000 times from PyPI. +use Requests internally. It has been installed over 60,000 times from PyPI. **Armin Ronacher** Requests is the perfect example how beautiful an API can be with the diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 099dabd7..4b3430f2 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -243,6 +243,27 @@ Then, we can make a request using our Pizza Auth:: +Streaming Requests +------------------ + +With ``requests.Response.iter_lines()`` you can easily iterate over streaming +APIs such as the `Twitter Streaming API `_. + +To use the Twitter Streaming API to track the keyword "requests": + +:: + + import requests + import json + + r = requests.post('https://stream.twitter.com/1/statuses/filter.json', + data={'track': 'requests'}, auth=('username', 'password')) + + for line in r.iter_lines(): + if line: # filter out keep-alive new lines + print json.loads(line) + + Verbose Logging --------------- diff --git a/requests/__init__.py b/requests/__init__.py index ee3173e1..847f0464 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -15,8 +15,8 @@ requests """ __title__ = 'requests' -__version__ = '0.8.6' -__build__ = 0x000806 +__version__ = '0.8.7' +__build__ = 0x000807 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' diff --git a/requests/async.py b/requests/async.py index c91025fc..6f8de380 100644 --- a/requests/async.py +++ b/requests/async.py @@ -36,6 +36,11 @@ def patched(f): kwargs['return_response'] = False kwargs['prefetch'] = True + config = kwargs.get('config', {}) + config.update(safe_mode=True) + + kwargs['config'] = config + return f(*args, **kwargs) return wrapped diff --git a/requests/models.py b/requests/models.py index e84c8f1a..1611bd07 100644 --- a/requests/models.py +++ b/requests/models.py @@ -18,13 +18,15 @@ from .structures import CaseInsensitiveDict from .status_codes import codes from .packages import oreos from .auth import HTTPBasicAuth, HTTPProxyAuth +from .packages.urllib3.response import HTTPResponse from .packages.urllib3.exceptions import MaxRetryError from .packages.urllib3.exceptions import SSLError as _SSLError from .packages.urllib3.exceptions import HTTPError as _HTTPError from .packages.urllib3 import connectionpool, poolmanager from .packages.urllib3.filepost import encode_multipart_formdata from .exceptions import ( - Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError) + ConnectionError, HTTPError, RequestException, Timeout, TooManyRedirects, + URLRequired) from .utils import ( get_encoding_from_headers, stream_decode_response_unicode, decode_gzip, stream_decode_gzip, guess_filename, requote_path) @@ -171,6 +173,9 @@ class Request(object): # Save cookies in Response. response.cookies = cookies + # No exceptions were harmed in the making of this request. + response.error = getattr(resp, 'error', None) + # Save original response for later. response.raw = resp @@ -439,32 +444,40 @@ class Request(object): self.headers['Cookie'] = cookie_header try: - # Send the request. - r = conn.urlopen( - method=self.method, - url=self.path_url, - body=body, - headers=self.headers, - redirect=False, - assert_same_host=False, - preload_content=False, - decode_content=False, - retries=self.config.get('max_retries', 0), - timeout=self.timeout, - ) - self.sent = True + # The inner try .. except re-raises certain exceptions as + # internal exception types; the outer suppresses exceptions + # when safe mode is set. + try: + # Send the request. + r = conn.urlopen( + method=self.method, + url=self.path_url, + body=body, + headers=self.headers, + redirect=False, + assert_same_host=False, + preload_content=False, + decode_content=False, + retries=self.config.get('max_retries', 0), + timeout=self.timeout, + ) + self.sent = True - - except MaxRetryError, e: - if not self.config.get('safe_mode', False): + except MaxRetryError, e: raise ConnectionError(e) - else: - r = None - except (_SSLError, _HTTPError), e: - if not self.config.get('safe_mode', False): + except (_SSLError, _HTTPError), e: raise Timeout('Request timed out.') + except RequestException, e: + if self.config.get('safe_mode', False): + # In safe mode, catch the exception and attach it to + # a blank urllib3.HTTPResponse object. + r = HTTPResponse() + r.error = e + else: + raise + self._build_response(r) # Response manipulation hook. @@ -595,18 +608,24 @@ class Response(object): ) def generate(): - chunk = [] + if self.raw is not None: + chunk = [] - while 1: - c = self.raw.read(1) - if not c: - break + while 1: + c = self.raw.read(1) + if not c: + break - if c in newlines: + if c in newlines: + yield ''.join(chunk) + chunk = [] + else: + chunk.append(c) + + # Yield the remainder, in case the response + # did not terminate with a newline + if chunk: yield ''.join(chunk) - chunk = [] - else: - chunk.append(c) self._content_consumed = True diff --git a/test_requests.py b/test_requests.py index 829d2a94..b3e55c8c 100755 --- a/test_requests.py +++ b/test_requests.py @@ -3,6 +3,7 @@ from __future__ import with_statement +import StringIO import time import os import unittest @@ -603,5 +604,45 @@ class RequestsTestSuite(unittest.TestCase): self.assertEqual(i, len_lines) + # Test 'dangling' fragment in responses that do not terminate in + # a newline. + quote = ( + '''Why will he not upon our fair request\n''' + '''Untent his person and share the air with us?''' + ) + + # Make a request and monkey-patch its contents + r = requests.get(httpbin('get')) + r.raw = StringIO.StringIO(quote) + + # Make sure iter_lines doesn't chop the trailing bit + lines = '\n'.join(r.iter_lines()) + self.assertEqual(lines, quote) + + def test_safe_mode(self): + + safe = requests.session(config=dict(safe_mode=True)) + + # Safe mode creates empty responses for failed requests. + # Iterating on these responses should produce empty sequences + r = safe.get('http://_/') + self.assertEquals(list(r.iter_lines()), []) + self.assertIsInstance(r.error, requests.exceptions.ConnectionError) + + r = safe.get('http://_/') + self.assertEquals(list(r.iter_content()), []) + self.assertIsInstance(r.error, requests.exceptions.ConnectionError) + + # When not in safe mode, should raise Timeout exception + with self.assertRaises(requests.exceptions.Timeout): + r = requests.get(httpbin('stream', '1000'), timeout=0.0001) + + # In safe mode, should return a blank response + r = requests.get(httpbin('stream', '1000'), timeout=0.0001, + config=dict(safe_mode=True)) + self.assertIsNone(r.content) + self.assertIsInstance(r.error, requests.exceptions.Timeout) + + if __name__ == '__main__': unittest.main()