Merge branch 'feature-content-streaming' of https://github.com/mitsuhiko/requests into feature/content-stream

Conflicts:
	AUTHORS
	HISTORY.rst
	docs/api.rst
	docs/community/faq.rst
	docs/user/intro.rst
	requests/api.py
	requests/models.py

AUTHORS

@@ -40,3 +40,7 @@ Patches and Suggestions
 - Mikko Ohtamaa
 - Den Shabalin
 - Daniel Miller <danielm@vs-networks.com>
+- Alejandro Giacometti
+- Rick Mak
+- Johan Bergström
+- Josselin Jacquard

AUTHORS.orig

@@ -1,40 +0,0 @@
-Requests is written and maintained by Kenneth Reitz and
-various contributors:
-
-Development Lead
-````````````````
-
-- Kenneth Reitz <me@kennethreitz.com>
-
-
-Patches and Suggestions
-```````````````````````
-
-- Various Pocoo Members
-- Chris Adams
-- Flavio Percoco Premoli
-- Dj Gilcrease
-- Justin Murphy
-- Rob Madole
-- Aram Dulyan
-- Johannes Gorset
-- 村山めがね (Megane Murayama)
-- James Rowe
-- Daniel Schauenberg
-- Zbigniew Siciarz
-- Daniele Tricoli 'Eriol'
-- Richard Boulton
-- Miguel Olivares <miguel@moliware.com>
-- Alberto Paro
-- Jérémy Bethmont
-- 潘旭 (Xu Pan)
-- Tamás Gulácsi
-- Rubén Abad
-- Peter Manser
-- Jeremy Selie
-<<<<<<< HEAD
-- Jens Diemer
-- Alex <@alopatin>
-=======
-- Tom Hogans <tomhsx@gmail.com>
->>>>>>> 0ed641a26ec2200de00e4bbf3d170c767375351e

HISTORY.rst

@@ -1,12 +1,17 @@
 History
 -------
 
+* Automatic decoding of unicode, based on HTTP Headers.
+* New ``decode_unicode`` setting
+* Removal of ``r.read/close`` methods
+* New ``r.fo`` interface for advanced response usage.*
+* Automatic expansion of parameterized headers
+
 0.6.3 (2011-10-13)
 ++++++++++++++++++
 
 * Beautiful ``requests.async`` module, for making async requests w/ gevent.
 
-
 0.6.2 (2011-10-09)
 ++++++++++++++++++
 

MANIFEST.in

@@ -1 +1 @@
-include README.rst LICENSE HISTORY.rst 
+include README.rst LICENSE HISTORY.rst test_requests.py

docs/community/faq.rst

@@ -10,6 +10,7 @@ Encoded Data?
 
 Requests automatically decompresses gzip-encoded responses, and does
 its best to decodes response content to unicode when possible.
+it's best to decodes response content to unicode when possible.
 
 You can get direct access to the raw response (and even the socket),
 if needed as well.

docs/user/install.rst

@@ -75,4 +75,4 @@ Ubuntu::
 
 Once you have ``libevent``, you can install ``gevent`` with ``pip``::
 
-    $ pip install gevent
+    $ pip install gevent

docs/user/intro.rst

@@ -49,3 +49,5 @@ Requests License
 
 
 
+Support for Python 3.x is planned.
+Support for Python 3.x is planned.

requests/api.py

@@ -15,7 +15,7 @@ import config
 from .models import Request, Response, AuthObject
 from .status_codes import codes
 from .hooks import dispatch_hook
-from .utils import cookiejar_from_dict
+from .utils import cookiejar_from_dict, header_expand
 
 
 __all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete')
@@ -24,8 +24,8 @@ def request(method, url,
     params=None, data=None, headers=None, cookies=None, files=None, auth=None,
     timeout=None, allow_redirects=False, proxies=None, hooks=None, return_response=True):
 
-    """Constructs and sends a :class:`Request <models.Request>`.
-    Returns :class:`Response <models.Response>` object.
+    """Constructs and sends a :class:`Request <Request>`.
+    Returns :class:`Response <Response>` object.
 
     :param method: method for the new :class:`Request` object.
     :param url: URL for the new :class:`Request` object.
@@ -40,11 +40,18 @@ def request(method, url,
     :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
     """
 
+    method = str(method).upper()
+
     if cookies is None:
         cookies = {}
 
     cookies = cookiejar_from_dict(cookies)
 
+    # Expand header values
+    if headers:
+        for k, v in headers.items() or {}:
+            headers[k] = header_expand(v)
+
     args = dict(
         method = method,
         url = url,
@@ -94,16 +101,27 @@ def get(url, **kwargs):
     :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
     :param auth: (optional) AuthObject to enable Basic HTTP Auth.
     :param timeout: (optional) Float describing the timeout of the request.
+    :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+    :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
+    :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
+    :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
+    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
+    :param timeout: (optional) Float describing the timeout of the request.
     :param allow_redirects: (optional) Boolean. Set to False to disable redirect following.
     :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+    :param **kwargs: Optional arguments that ``request`` takes.
     """
 
+    return request('GET', url, **kwargs)
     kwargs.setdefault('allow_redirects', True)
     return request('GET', url, **kwargs)
+    if "allow_redirects" not in kwargs:
+        kwargs["allow_redirects"] = True
+
+    return request('get', url, **kwargs)
 
 
 def head(url, **kwargs):
-
     """Sends a HEAD request. Returns :class:`Response` object.
 
     :param url: URL for the new :class:`Request` object.
@@ -112,31 +130,35 @@ def head(url, **kwargs):
     :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
     :param auth: (optional) AuthObject to enable Basic HTTP Auth.
     :param timeout: (optional) Float describing the timeout of the request.
+    :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+    :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
+    :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
+    :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
+    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
+    :param timeout: (optional) Float describing the timeout of the request.
     :param allow_redirects: (optional) Boolean. Set to False to disable redirect following.
     :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+    :param **kwargs: Optional arguments that ``request`` takes.
     """
 
+    return request('HEAD', url, **kwargs)
     kwargs.setdefault('allow_redirects', True)
     return request('HEAD', url, **kwargs)
+    if "allow_redirects" not in kwargs:
+        kwargs["allow_redirects"] = True
+
+    return request('head', url, **kwargs)
 
 
 def post(url, data='', **kwargs):
-
     """Sends a POST request. Returns :class:`Response` object.
 
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
-    :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
-    :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
-    :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
-    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
-    :param timeout: (optional) Float describing the timeout of the request.
-    :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
-    :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
-    :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+    :param **kwargs: Optional arguments that ``request`` takes.
     """
 
-    return request('POST', url, data=data, **kwargs)
+    return request('post', url, data=data, **kwargs)
 
 
 def put(url, data='', **kwargs):
@@ -144,17 +166,10 @@ def put(url, data='', **kwargs):
 
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
-    :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
-    :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
-    :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
-    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
-    :param timeout: (optional) Float describing the timeout of the request.
-    :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
-    :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
-    :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+    :param **kwargs: Optional arguments that ``request`` takes.
     """
 
-    return request('PUT', url, data=data, **kwargs)
+    return request('put', url, data=data, **kwargs)
 
 
 def patch(url, data='', **kwargs):
@@ -162,31 +177,17 @@ def patch(url, data='', **kwargs):
 
     :param url: URL for the new :class:`Request` object.
     :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
-    :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
-    :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
-    :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
-    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
-    :param timeout: (optional) Float describing the timeout of the request.
-    :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
-    :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
-    :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+    :param **kwargs: Optional arguments that ``request`` takes.
     """
 
-    return request('PATCH', url, **kwargs)
+    return request('patch', url, **kwargs)
 
 
 def delete(url, **kwargs):
-
     """Sends a DELETE request. Returns :class:`Response` object.
 
     :param url: URL for the new :class:`Request` object.
-    :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
-    :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
-    :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
-    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
-    :param timeout: (optional) Float describing the timeout of the request.
-    :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
-    :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
+    :param **kwargs: Optional arguments that ``request`` takes.
     """
 
-    return request('DELETE', url, **kwargs)
+    return request('delete', url, **kwargs)

requests/config.py

@@ -62,6 +62,7 @@ settings.proxies = None
 settings.verbose = None
 settings.timeout = None
 settings.max_redirects = 30
+settings.decode_unicode = True
 
 #: Use socket.setdefaulttimeout() as fallback?
 settings.timeout_fallback = True

requests/core.py

@@ -19,9 +19,11 @@ __license__ = 'ISC'
 __copyright__ = 'Copyright 2011 Kenneth Reitz'
 
 
-from models import HTTPError
+from models import HTTPError, Request, Response
 from api import *
 from exceptions import *
 from sessions import session
 from status_codes import codes
-from config import settings
+from config import settings
+
+import utils

requests/models.py

@@ -9,8 +9,10 @@ requests.models
 import urllib
 import urllib2
 import socket
+import codecs
 import zlib
 
+
 from urllib2 import HTTPError
 from urlparse import urlparse, urlunparse, urljoin
 from datetime import datetime
@@ -20,9 +22,9 @@ from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicA
 from .structures import CaseInsensitiveDict
 from .packages.poster.encode import multipart_encode
 from .packages.poster.streaminghttp import register_openers, get_handlers
-from .utils import dict_from_cookiejar
-from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects
+from .utils import dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip
 from .status_codes import codes
+from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects
 
 
 REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
@@ -30,7 +32,7 @@ REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
 
 
 class Request(object):
-    """The :class:`Request <models.Request>` object. It carries out all functionality of
+    """The :class:`Request <Request>` object. It carries out all functionality of
     Requests. Recommended interface is with the Requests functions.
     """
 
@@ -46,24 +48,24 @@ class Request(object):
         #: Request URL.
         self.url = url
 
-        #: Dictonary of HTTP Headers to attach to the :class:`Request <models.Request>`.
+        #: Dictonary of HTTP Headers to attach to the :class:`Request <Request>`.
         self.headers = headers
 
         #: Dictionary of files to multipart upload (``{filename: content}``).
         self.files = files
 
-        #: HTTP Method to use. Available: GET, HEAD, PUT, POST, DELETE.
+        #: HTTP Method to use.
         self.method = method
 
         #: Dictionary or byte of request body data to attach to the
-        #: :class:`Request <models.Request>`.
+        #: :class:`Request <Request>`.
         self.data = None
 
         #: Dictionary or byte of querystring data to attach to the
-        #: :class:`Request <models.Request>`.
+        #: :class:`Request <Request>`.
         self.params = None
 
-        #: True if :class:`Request <models.Request>` is part of a redirect chain (disables history
+        #: True if :class:`Request <Request>` is part of a redirect chain (disables history
         #: and HTTPError storage).
         self.redirect = redirect
 
@@ -76,7 +78,7 @@ class Request(object):
         self.data, self._enc_data = self._encode_params(data)
         self.params, self._enc_params = self._encode_params(params)
 
-        #: :class:`Response <models.Response>` instance, containing
+        #: :class:`Response <Response>` instance, containing
         #: content and metadata of HTTP Response, once :attr:`sent <send>`.
         self.response = Response()
 
@@ -85,10 +87,10 @@ class Request(object):
         if not auth:
             auth = auth_manager.get_auth(self.url)
 
-        #: :class:`AuthObject` to attach to :class:`Request <models.Request>`.
+        #: :class:`AuthObject` to attach to :class:`Request <Request>`.
         self.auth = auth
 
-        #: CookieJar to attach to :class:`Request <models.Request>`.
+        #: CookieJar to attach to :class:`Request <Request>`.
         self.cookiejar = cookiejar
 
         #: True if Request has been sent.
@@ -134,9 +136,16 @@ class Request(object):
             _handlers.append(urllib2.HTTPCookieProcessor(self.cookiejar))
 
         if self.auth:
-            if not isinstance(self.auth.handler, (urllib2.AbstractBasicAuthHandler, urllib2.AbstractDigestAuthHandler)):
+            if not isinstance(self.auth.handler,
+                (urllib2.AbstractBasicAuthHandler,
+                urllib2.AbstractDigestAuthHandler)):
+
                 # TODO: REMOVE THIS COMPLETELY
-                auth_manager.add_password(self.auth.realm, self.url, self.auth.username, self.auth.password)
+                auth_manager.add_password(
+                    self.auth.realm, self.url,
+                    self.auth.username,
+                    self.auth.password)
+
                 self.auth.handler = self.auth.handler(auth_manager)
                 auth_manager.add_auth(self.url, self.auth)
 
@@ -168,7 +177,10 @@ class Request(object):
 
 
     def _build_response(self, resp, is_error=False):
-        """Build internal :class:`Response <models.Response>` object from given response."""
+        """Build internal :class:`Response <Response>` object
+        from given response.
+        """
+
 
         def build(resp):
 
@@ -177,9 +189,7 @@ class Request(object):
 
             try:
                 response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None))
-                response.read = resp.read
-                response._resp = resp
-                response._close = resp.close
+                response.fo = resp
 
                 if self.cookiejar:
                     response.cookies = dict_from_cookiejar(self.cookiejar)
@@ -204,10 +214,15 @@ class Request(object):
 
             while (
                 ('location' in r.headers) and
+                ((self.method in ('GET', 'HEAD')) or
+                (r.status_code is codes.see_other) or
+                (self.allow_redirects))
                 ((r.status_code is codes.see_other) or (self.allow_redirects))
+                ((r.status_code is codes.see_other) or
+                (self.allow_redirects))
             ):
 
-                r.close()
+                r.fo.close()
 
                 if not len(history) < settings.max_redirects:
                     raise TooManyRedirects()
@@ -256,8 +271,8 @@ class Request(object):
 
         Otherwise, assumes the data is already encoded appropriately, and
         returns it twice.
-
         """
+
         if hasattr(data, 'items'):
             result = []
             for k, vs in data.items():
@@ -301,7 +316,6 @@ class Request(object):
         """
 
         self._checks()
-        success = False
 
         # Logging
         if settings.verbose:
@@ -367,7 +381,6 @@ class Request(object):
 
                 self._build_response(why, is_error=True)
 
-
             else:
                 self._build_response(resp)
                 self.response.ok = True
@@ -378,37 +391,46 @@ class Request(object):
         return self.sent
 
 
-
 class Response(object):
-    """The core :class:`Response <models.Response>` object. All
-    :class:`Request <models.Request>` objects contain a
-    :class:`response <models.Response>` attribute, which is an instance
+    """The core :class:`Response <Response>` object. All
+    :class:`Request <Request>` objects contain a
+    :class:`response <Response>` attribute, which is an instance
     of this class.
     """
 
     def __init__(self):
-        #: Raw content of the response, in bytes.
-        #: If ``content-encoding`` of response was set to ``gzip``, the
-        #: response data will be automatically deflated.
+
         self._content = None
+        self._content_consumed = False
+
         #: Integer Code of responded HTTP Status.
         self.status_code = None
+
         #: Case-insensitive Dictionary of Response Headers.
         #: For example, ``headers['content-encoding']`` will return the
         #: value of a ``'Content-Encoding'`` response header.
         self.headers = CaseInsensitiveDict()
+
+        #: File-like object representation of response (for advanced usage).
+        self.fo = None
+
         #: Final URL location of Response.
         self.url = None
+
         #: True if no :attr:`error` occured.
         self.ok = False
+
         #: Resulting :class:`HTTPError` of request, if one occured.
         self.error = None
-        #: A list of :class:`Response <models.Response>` objects from
+
+        #: A list of :class:`Response <Response>` objects from
         #: the history of the Request. Any redirect responses will end
         #: up here.
         self.history = []
-        #: The Request that created the Response.
+
+        #: The :class:`Request <Request>` that created the Response.
         self.request = None
+
         #: A dictionary of Cookies the server sent back.
         self.cookies = None
 
@@ -419,23 +441,65 @@ class Response(object):
 
     def __nonzero__(self):
         """Returns true if :attr:`status_code` is 'OK'."""
+
         return not self.error
 
+    def iter_content(self, chunk_size=10 * 1024, decode_unicode=None):
+        """Iterates over the response data.  This avoids reading the content
+        at once into memory for large responses.  The chunk size is the number
+        of bytes it should read into memory.  This is not necessarily the
+        length of each item returned as decoding can take place.
+        """
+        if self._content_consumed:
+            raise RuntimeError('The content for this response was '
+                               'already consumed')
 
-    def __getattr__(self, name):
-        """Read and returns the full stream when accessing to :attr: `content`"""
-        if name == 'content':
-            if self._content is not None:
-                return self._content
-            self._content = self.read()
-            if self.headers.get('content-encoding', '') == 'gzip':
-                try:
-                    self._content = zlib.decompress(self._content, 16+zlib.MAX_WBITS)
-                except zlib.error:
-                    pass
+        def generate():
+            while 1:
+                chunk = self.fo.read(chunk_size)
+                if not chunk:
+                    break
+                yield chunk
+            self._content_consumed = True
+        gen = generate()
+        if 'gzip' in self.headers.get('content-encoding', ''):
+            gen = stream_decode_gzip(gen)
+        if decode_unicode is None:
+            decode_unicode = settings.decode_unicode
+        if decode_unicode:
+            gen = stream_decode_response_unicode(gen, self)
+        return gen
+
+    @property
+    def content(self):
+        """Content of the response, in bytes or unicode
+        (if available).
+        """
+
+        if self._content is not None:
             return self._content
-        else:
-            raise AttributeError
+
+        if self._content_consumed:
+            raise RuntimeError('The content for this response was '
+                               'already consumed')
+
+        # Read the contents.
+        self._content = self.fo.read()
+
+        # Decode GZip'd content.
+        if 'gzip' in self.headers.get('content-encoding', ''):
+            try:
+                self._content = decode_gzip(self._content)
+            except zlib.error:
+                pass
+
+        # Decode unicode content.
+        if settings.decode_unicode:
+            self._content = get_unicode_from_response(self)
+
+        self._content_consumed = True
+        return self._content
+
 
     def raise_for_status(self):
         """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured."""
@@ -443,10 +507,6 @@ class Response(object):
             raise self.error
 
 
-    def close(self):
-        if self._resp.fp is not None and hasattr(self._resp.fp, '_sock'):
-            self._resp.fp._sock.recv = None
-        self._close()
 
 class AuthManager(object):
     """Requests Authentication Manager."""

requests/utils.py

@@ -9,15 +9,69 @@ that are also useful for external consumption.
 
 """
 
+import cgi
+import codecs
 import cookielib
+import re
+import zlib
 
 
-def dict_from_cookiejar(cookiejar):
-    """Returns a key/value dictionary from a CookieJar."""
+def header_expand(headers):
+    """Returns an HTTP Header value string from a dictionary.
+
+    Example expansion::
+
+        {'text/x-dvi': {'q': '.8', 'mxb': '100000', 'mxt': '5.0'}, 'text/x-c': {}}
+        # Accept: text/x-dvi; q=.8; mxb=100000; mxt=5.0, text/x-c
+
+        (('text/x-dvi', {'q': '.8', 'mxb': '100000', 'mxt': '5.0'}), ('text/x-c', {}))
+        # Accept: text/x-dvi; q=.8; mxb=100000; mxt=5.0, text/x-c
+    """
+
+    collector = []
+
+    if isinstance(headers, dict):
+        headers = headers.items()
+
+    elif isinstance(headers, basestring):
+        return headers
+
+    for i, (value, params) in enumerate(headers):
+
+        _params = []
+
+        for (p_k, p_v) in params.items():
+
+            _params.append('%s=%s' % (p_k, p_v))
+
+        collector.append(value)
+        collector.append('; ')
+
+        if len(params):
+
+            collector.append('; '.join(_params))
+
+            if not len(headers) == i+1:
+                collector.append(', ')
+
+
+    # Remove trailing seperators.
+    if collector[-1] in (', ', '; '):
+        del collector[-1]
+
+    return ''.join(collector)
+
+
+
+def dict_from_cookiejar(cj):
+    """Returns a key/value dictionary from a CookieJar.
+
+    :param cj: CookieJar object to extract cookies from.
+    """
 
     cookie_dict = {}
 
-    for _, cookies in cookiejar._cookies.items():
+    for _, cookies in cj._cookies.items():
         for _, cookies in cookies.items():
             for cookie in cookies.values():
                 # print cookie
@@ -27,7 +81,10 @@ def dict_from_cookiejar(cookiejar):
 
 
 def cookiejar_from_dict(cookie_dict):
-    """Returns a CookieJar from a key/value dictionary."""
+    """Returns a CookieJar from a key/value dictionary.
+
+    :param cookie_dict: Dict of key/values to insert into CookieJar.
+    """
 
     # return cookiejar if one was passed in
     if isinstance(cookie_dict, cookielib.CookieJar):
@@ -42,7 +99,11 @@ def cookiejar_from_dict(cookie_dict):
 
 
 def add_dict_to_cookiejar(cj, cookie_dict):
-    """Returns a CookieJar from a key/value dictionary."""
+    """Returns a CookieJar from a key/value dictionary.
+
+    :param cj: CookieJar to insert cookies into.
+    :param cookie_dict: Dict of key/values to insert into CookieJar.
+    """
 
     for k, v in cookie_dict.items():
 
@@ -70,3 +131,124 @@ def add_dict_to_cookiejar(cj, cookie_dict):
         cj.set_cookie(cookie)
 
     return cj
+
+
+def get_encodings_from_content(content):
+    """Returns encodings from given content string.
+
+    :param content: bytestring to extract encodings from.
+    """
+
+    charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)
+
+    return charset_re.findall(content)
+
+
+def get_encoding_from_headers(headers):
+    """Returns encodings from given HTTP Header Dict.
+
+    :param headers: dictionary to extract encoding from.
+    """
+
+    content_type = headers.get('content-type')
+
+    if not content_type:
+        return None
+
+    content_type, params = cgi.parse_header(content_type)
+
+    if 'charset' in params:
+        return params['charset'].strip("'\"")
+
+
+def unicode_from_html(content):
+    """Attempts to decode an HTML string into unicode.
+    If unsuccessful, the original content is returned.
+    """
+
+    encodings = get_encodings_from_content(content)
+
+    for encoding in encodings:
+
+        try:
+            return unicode(content, encoding)
+        except (UnicodeError, TypeError):
+            pass
+
+        return content
+
+
+def stream_decode_response_unicode(iterator, r):
+    """Stream decodes a iterator."""
+    encoding = get_encoding_from_headers(r.headers)
+    if encoding is None:
+        for item in iterator:
+            yield item
+        return
+
+    decoder = codecs.getincrementaldecoder(encoding)(errors='replace')
+    for chunk in iterator:
+        rv = decoder.decode(chunk)
+        if rv:
+            yield rv
+    rv = decoder.decode('', final=True)
+    if rv:
+        yield rv
+
+
+def get_unicode_from_response(r):
+    """Returns the requested content back in unicode.
+
+    :param r: Reponse object to get unicode content from.
+
+    Tried:
+
+    1. charset from content-type
+
+    2. every encodings from ``<meta ... charset=XXX>``
+
+    3. fall back and replace all unicode characters
+
+    """
+
+    tried_encodings = []
+
+    # Try charset from content-type
+    encoding = get_encoding_from_headers(r.headers)
+
+    if encoding:
+        try:
+            return unicode(r.content, encoding)
+        except UnicodeError:
+            tried_encodings.append(encoding)
+
+    # Fall back:
+    try:
+        return unicode(r.content, encoding, errors='replace')
+    except TypeError:
+        return r.content
+
+
+def decode_gzip(content):
+    """Return gzip-decoded string.
+
+    :param content: bytestring to gzip-decode.
+    """
+
+    return zlib.decompress(content, 16 + zlib.MAX_WBITS)
+
+
+def stream_decode_gzip(iterator):
+    """Stream decodes a gzip-encoded iterator"""
+    try:
+        dec = zlib.decompressobj(16 + zlib.MAX_WBITS)
+        for chunk in iterator:
+            rv = dec.decompress(chunk)
+            if rv:
+                yield rv
+        buf = dec.decompress('')
+        rv = buf + dec.flush()
+        if rv:
+            yield rv
+    except zlib.error:
+        pass

test_requests.py

@@ -62,6 +62,13 @@ class RequestsTestSuite(unittest.TestCase):
         r = requests.get(httpbin('/'))
         self.assertEqual(r.status_code, 200)
 
+    def test_HTTP_302_ALLOW_REDIRECT_GET(self):
+	r = requests.get(httpbin('redirect', '1'))
+	self.assertEqual(r.status_code, 200)
+
+    def test_HTTP_302_GET(self):
+	r = requests.get(httpbin('redirect', '1'), allow_redirects=False)
+	self.assertEqual(r.status_code, 302)
 
     def test_HTTPS_200_OK_GET(self):
         r = requests.get(httpsbin('/'))
@@ -490,6 +497,5 @@ class RequestsTestSuite(unittest.TestCase):
         self.assertEqual(r2.status_code, 200)
 
 
-
 if __name__ == '__main__':
     unittest.main()