Merge branch 'develop'

AUTHORS.rst

@@ -65,4 +65,6 @@ Patches and Suggestions
 - Idan Gazit
 - Ed Summers
 - Chris Van Horne
-- Christopher Davis
+- Christopher Davis
+- Ori Livneh
+- Jason Emerick

HISTORY.rst

@@ -1,13 +1,20 @@
 History
 -------
 
+0.8.7 (2011-12-24)
+++++++++++++++++++
+
+* iter_lines last-line truncation fix
+* Force safe_mode for async requests
+* Handle safe_mode exceptions more consistently
+* Fix iteration on null responses in safe_mode
+
 0.8.6 (2011-12-18)
 ++++++++++++++++++
 
 * Socket timeout fixes.
 * Proxy Authorization support.
 
-
 0.8.5 (2011-12-14)
 ++++++++++++++++++
 

docs/index.rst

@@ -38,7 +38,7 @@ Testimonials
 `Twitter, Inc <http://twitter.com>`_,
 `Readability <http://readability.com>`_, and
 Federal US Institutions
-use Requests internally. It has been installed over 45,000 times from PyPI.
+use Requests internally. It has been installed over 60,000 times from PyPI.
 
 **Armin Ronacher**
     Requests is the perfect example how beautiful an API can be with the

docs/user/advanced.rst

@@ -243,6 +243,27 @@ Then, we can make a request using our Pizza Auth::
     <Response [200]>
 
 
+Streaming Requests
+------------------
+
+With ``requests.Response.iter_lines()`` you can easily iterate over streaming 
+APIs such as the `Twitter Streaming API <https://dev.twitter.com/docs/streaming-api>`_.
+
+To use the Twitter Streaming API to track the keyword "requests":
+
+::
+
+    import requests
+    import json
+
+    r = requests.post('https://stream.twitter.com/1/statuses/filter.json',
+        data={'track': 'requests'}, auth=('username', 'password'))
+
+    for line in r.iter_lines():
+	    if line: # filter out keep-alive new lines
+		    print json.loads(line)
+
+
 Verbose Logging
 ---------------
 

requests/__init__.py

@@ -15,8 +15,8 @@ requests
 """
 
 __title__ = 'requests'
-__version__ = '0.8.6'
-__build__ = 0x000806
+__version__ = '0.8.7'
+__build__ = 0x000807
 __author__ = 'Kenneth Reitz'
 __license__ = 'ISC'
 __copyright__ = 'Copyright 2011 Kenneth Reitz'

requests/async.py

@@ -36,6 +36,11 @@ def patched(f):
         kwargs['return_response'] = False
         kwargs['prefetch'] = True
 
+        config = kwargs.get('config', {})
+        config.update(safe_mode=True)
+
+        kwargs['config'] = config
+
         return f(*args, **kwargs)
 
     return wrapped

requests/models.py

@@ -18,13 +18,15 @@ from .structures import CaseInsensitiveDict
 from .status_codes import codes
 from .packages import oreos
 from .auth import HTTPBasicAuth, HTTPProxyAuth
+from .packages.urllib3.response import HTTPResponse
 from .packages.urllib3.exceptions import MaxRetryError
 from .packages.urllib3.exceptions import SSLError as _SSLError
 from .packages.urllib3.exceptions import HTTPError as _HTTPError
 from .packages.urllib3 import connectionpool, poolmanager
 from .packages.urllib3.filepost import encode_multipart_formdata
 from .exceptions import (
-    Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError)
+    ConnectionError, HTTPError, RequestException, Timeout, TooManyRedirects,
+    URLRequired)
 from .utils import (
     get_encoding_from_headers, stream_decode_response_unicode,
     decode_gzip, stream_decode_gzip, guess_filename, requote_path)
@@ -171,6 +173,9 @@ class Request(object):
                 # Save cookies in Response.
                 response.cookies = cookies
 
+                # No exceptions were harmed in the making of this request.
+                response.error = getattr(resp, 'error', None)
+
             # Save original response for later.
             response.raw = resp
 
@@ -439,32 +444,40 @@ class Request(object):
                     self.headers['Cookie'] = cookie_header
 
             try:
-                # Send the request.
-                r = conn.urlopen(
-                    method=self.method,
-                    url=self.path_url,
-                    body=body,
-                    headers=self.headers,
-                    redirect=False,
-                    assert_same_host=False,
-                    preload_content=False,
-                    decode_content=False,
-                    retries=self.config.get('max_retries', 0),
-                    timeout=self.timeout,
-                )
-                self.sent = True
+                # The inner try .. except re-raises certain exceptions as
+                # internal exception types; the outer suppresses exceptions
+                # when safe mode is set.
+                try:
+                    # Send the request.
+                    r = conn.urlopen(
+                        method=self.method,
+                        url=self.path_url,
+                        body=body,
+                        headers=self.headers,
+                        redirect=False,
+                        assert_same_host=False,
+                        preload_content=False,
+                        decode_content=False,
+                        retries=self.config.get('max_retries', 0),
+                        timeout=self.timeout,
+                    )
+                    self.sent = True
 
-
-            except MaxRetryError, e:
-                if not self.config.get('safe_mode', False):
+                except MaxRetryError, e:
                     raise ConnectionError(e)
-                else:
-                    r = None
 
-            except (_SSLError, _HTTPError), e:
-                if not self.config.get('safe_mode', False):
+                except (_SSLError, _HTTPError), e:
                     raise Timeout('Request timed out.')
 
+            except RequestException, e:
+                if self.config.get('safe_mode', False):
+                    # In safe mode, catch the exception and attach it to 
+                    # a blank urllib3.HTTPResponse object.
+                    r = HTTPResponse()
+                    r.error = e
+                else:
+                    raise
+
             self._build_response(r)
 
             # Response manipulation hook.
@@ -595,18 +608,24 @@ class Response(object):
             )
 
         def generate():
-            chunk = []
+            if self.raw is not None:
+                chunk = []
 
-            while 1:
-                c = self.raw.read(1)
-                if not c:
-                    break
+                while 1:
+                    c = self.raw.read(1)
+                    if not c:
+                        break
 
-                if c in newlines:
+                    if c in newlines:
+                        yield ''.join(chunk)
+                        chunk = []
+                    else:
+                        chunk.append(c)
+
+                # Yield the remainder, in case the response
+                # did not terminate with a newline
+                if chunk:
                     yield ''.join(chunk)
-                    chunk = []
-                else:
-                    chunk.append(c)
 
             self._content_consumed = True
 

test_requests.py

@@ -3,6 +3,7 @@
 
 from __future__ import with_statement
 
+import StringIO
 import time
 import os
 import unittest
@@ -603,5 +604,45 @@ class RequestsTestSuite(unittest.TestCase):
 
             self.assertEqual(i, len_lines)
 
+        # Test 'dangling' fragment in responses that do not terminate in
+        # a newline.
+        quote = (
+            '''Why will he not upon our fair request\n'''
+            '''Untent his person and share the air with us?'''
+        )
+
+        # Make a request and monkey-patch its contents
+        r = requests.get(httpbin('get'))
+        r.raw = StringIO.StringIO(quote)
+
+        # Make sure iter_lines doesn't chop the trailing bit
+        lines = '\n'.join(r.iter_lines())
+        self.assertEqual(lines, quote)
+
+    def test_safe_mode(self):
+
+        safe = requests.session(config=dict(safe_mode=True))
+
+        # Safe mode creates empty responses for failed requests.
+        # Iterating on these responses should produce empty sequences
+        r = safe.get('http://_/')
+        self.assertEquals(list(r.iter_lines()), [])
+        self.assertIsInstance(r.error, requests.exceptions.ConnectionError)
+
+        r = safe.get('http://_/')
+        self.assertEquals(list(r.iter_content()), [])
+        self.assertIsInstance(r.error, requests.exceptions.ConnectionError)
+
+        # When not in safe mode, should raise Timeout exception
+        with self.assertRaises(requests.exceptions.Timeout):
+            r = requests.get(httpbin('stream', '1000'), timeout=0.0001)
+
+        # In safe mode, should return a blank response
+        r = requests.get(httpbin('stream', '1000'), timeout=0.0001,
+                config=dict(safe_mode=True))
+        self.assertIsNone(r.content)
+        self.assertIsInstance(r.error, requests.exceptions.Timeout)
+
+
 if __name__ == '__main__':
     unittest.main()