Merge pull request #1425 from Lukasa/stream

Use new urllib3 'stream' parameter.

requests/models.py

@@ -537,11 +537,18 @@ class Response(object):
             return iter_slices(self._content, chunk_size)
 
         def generate():
-            while 1:
-                chunk = self.raw.read(chunk_size, decode_content=True)
-                if not chunk:
-                    break
-                yield chunk
+            try:
+                # Special case for urllib3.
+                for chunk in self.raw.stream(chunk_size, decode_content=True):
+                    yield chunk
+            except AttributeError:
+                # Standard file-like object.
+                while 1:
+                    chunk = self.raw.read(chunk_size, decode_content=True)
+                    if not chunk:
+                        break
+                    yield chunk
+
             self._content_consumed = True
 
         gen = generate()

requests/packages/urllib3/response.py

@@ -11,6 +11,7 @@ import io
 
 from .exceptions import DecodeError
 from .packages.six import string_types as basestring, binary_type
+from .util import is_fp_closed
 
 
 log = logging.getLogger(__name__)
@@ -201,6 +202,29 @@ class HTTPResponse(io.IOBase):
             if self._original_response and self._original_response.isclosed():
                 self.release_conn()
 
+    def stream(self, amt=2**16, decode_content=None):
+        """
+        A generator wrapper for the read() method. A call will block until
+        ``amt`` bytes have been read from the connection or until the
+        connection is closed.
+
+        :param amt:
+            How much of the content to read. The generator will return up to
+            much data per iteration, but may return less. This is particularly
+            likely when using compressed data. However, the empty string will
+            never be returned.
+
+        :param decode_content:
+            If True, will attempt to decode the body based on the
+            'content-encoding' header.
+        """
+        while not is_fp_closed(self._fp):
+            data = self.read(amt=amt, decode_content=decode_content)
+
+            if data:
+                yield data
+
+
     @classmethod
     def from_httplib(ResponseCls, r, **response_kw):
         """

requests/packages/urllib3/util.py

@@ -31,7 +31,6 @@ try:  # Test for SSL features
 except ImportError:
     pass
 
-
 from .packages import six
 from .exceptions import LocationParseError, SSLError
 
@@ -341,6 +340,20 @@ def assert_fingerprint(cert, fingerprint):
                        .format(hexlify(fingerprint_bytes),
                                hexlify(cert_digest)))
 
+def is_fp_closed(obj):
+    """
+    Checks whether a given file-like object is closed.
+
+    :param obj:
+        The file-like object to check.
+    """
+    if hasattr(obj, 'fp'):
+        # Object is a container for another file-like object that gets released
+        # on exhaustion (e.g. HTTPResponse)
+        return obj.fp is None
+
+    return obj.closed
+
 
 if SSLContext is not None:  # Python 3.2+
     def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,