diff --git a/AUTHORS b/AUTHORS index 8dccb461..bac93676 100644 --- a/AUTHORS +++ b/AUTHORS @@ -40,3 +40,7 @@ Patches and Suggestions - Mikko Ohtamaa - Den Shabalin - Daniel Miller +- Alejandro Giacometti +- Rick Mak +- Johan Bergström +- Josselin Jacquard diff --git a/AUTHORS.orig b/AUTHORS.orig deleted file mode 100644 index dd88224b..00000000 --- a/AUTHORS.orig +++ /dev/null @@ -1,40 +0,0 @@ -Requests is written and maintained by Kenneth Reitz and -various contributors: - -Development Lead -```````````````` - -- Kenneth Reitz - - -Patches and Suggestions -``````````````````````` - -- Various Pocoo Members -- Chris Adams -- Flavio Percoco Premoli -- Dj Gilcrease -- Justin Murphy -- Rob Madole -- Aram Dulyan -- Johannes Gorset -- 村山めがね (Megane Murayama) -- James Rowe -- Daniel Schauenberg -- Zbigniew Siciarz -- Daniele Tricoli 'Eriol' -- Richard Boulton -- Miguel Olivares -- Alberto Paro -- Jérémy Bethmont -- 潘旭 (Xu Pan) -- Tamás Gulácsi -- Rubén Abad -- Peter Manser -- Jeremy Selie -<<<<<<< HEAD -- Jens Diemer -- Alex <@alopatin> -======= -- Tom Hogans ->>>>>>> 0ed641a26ec2200de00e4bbf3d170c767375351e diff --git a/HISTORY.rst b/HISTORY.rst index 229d8fcc..c5c711f9 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,12 +1,17 @@ History ------- +* Automatic decoding of unicode, based on HTTP Headers. +* New ``decode_unicode`` setting +* Removal of ``r.read/close`` methods +* New ``r.fo`` interface for advanced response usage.* +* Automatic expansion of parameterized headers + 0.6.3 (2011-10-13) ++++++++++++++++++ * Beautiful ``requests.async`` module, for making async requests w/ gevent. - 0.6.2 (2011-10-09) ++++++++++++++++++ diff --git a/MANIFEST.in b/MANIFEST.in index 94c50f70..39fbb994 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include README.rst LICENSE HISTORY.rst \ No newline at end of file +include README.rst LICENSE HISTORY.rst test_requests.py diff --git a/docs/community/faq.rst b/docs/community/faq.rst index 91ca06b2..0cda6591 100644 --- a/docs/community/faq.rst +++ b/docs/community/faq.rst @@ -10,6 +10,7 @@ Encoded Data? Requests automatically decompresses gzip-encoded responses, and does its best to decodes response content to unicode when possible. +it's best to decodes response content to unicode when possible. You can get direct access to the raw response (and even the socket), if needed as well. diff --git a/docs/user/install.rst b/docs/user/install.rst index 6179abdc..aef598dc 100644 --- a/docs/user/install.rst +++ b/docs/user/install.rst @@ -75,4 +75,4 @@ Ubuntu:: Once you have ``libevent``, you can install ``gevent`` with ``pip``:: - $ pip install gevent \ No newline at end of file + $ pip install gevent diff --git a/docs/user/intro.rst b/docs/user/intro.rst index c2b6bf36..0a168d1b 100644 --- a/docs/user/intro.rst +++ b/docs/user/intro.rst @@ -49,3 +49,5 @@ Requests License +Support for Python 3.x is planned. +Support for Python 3.x is planned. diff --git a/requests/api.py b/requests/api.py index 89eb44cc..54aeff88 100644 --- a/requests/api.py +++ b/requests/api.py @@ -15,7 +15,7 @@ import config from .models import Request, Response, AuthObject from .status_codes import codes from .hooks import dispatch_hook -from .utils import cookiejar_from_dict +from .utils import cookiejar_from_dict, header_expand __all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete') @@ -24,8 +24,8 @@ def request(method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=False, proxies=None, hooks=None, return_response=True): - """Constructs and sends a :class:`Request `. - Returns :class:`Response ` object. + """Constructs and sends a :class:`Request `. + Returns :class:`Response ` object. :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. @@ -40,11 +40,18 @@ def request(method, url, :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ + method = str(method).upper() + if cookies is None: cookies = {} cookies = cookiejar_from_dict(cookies) + # Expand header values + if headers: + for k, v in headers.items() or {}: + headers[k] = header_expand(v) + args = dict( method = method, url = url, @@ -94,16 +101,27 @@ def get(url, **kwargs): :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to False to disable redirect following. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param **kwargs: Optional arguments that ``request`` takes. """ + return request('GET', url, **kwargs) kwargs.setdefault('allow_redirects', True) return request('GET', url, **kwargs) + if "allow_redirects" not in kwargs: + kwargs["allow_redirects"] = True + + return request('get', url, **kwargs) def head(url, **kwargs): - """Sends a HEAD request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. @@ -112,31 +130,35 @@ def head(url, **kwargs): :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to False to disable redirect following. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param **kwargs: Optional arguments that ``request`` takes. """ + return request('HEAD', url, **kwargs) kwargs.setdefault('allow_redirects', True) return request('HEAD', url, **kwargs) + if "allow_redirects" not in kwargs: + kwargs["allow_redirects"] = True + + return request('head', url, **kwargs) def post(url, data='', **kwargs): - """Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. - :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. - :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. - :param auth: (optional) AuthObject to enable Basic HTTP Auth. - :param timeout: (optional) Float describing the timeout of the request. - :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. - :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. - :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param **kwargs: Optional arguments that ``request`` takes. """ - return request('POST', url, data=data, **kwargs) + return request('post', url, data=data, **kwargs) def put(url, data='', **kwargs): @@ -144,17 +166,10 @@ def put(url, data='', **kwargs): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. - :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. - :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. - :param auth: (optional) AuthObject to enable Basic HTTP Auth. - :param timeout: (optional) Float describing the timeout of the request. - :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. - :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. - :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param **kwargs: Optional arguments that ``request`` takes. """ - return request('PUT', url, data=data, **kwargs) + return request('put', url, data=data, **kwargs) def patch(url, data='', **kwargs): @@ -162,31 +177,17 @@ def patch(url, data='', **kwargs): :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. - :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. - :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. - :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. - :param auth: (optional) AuthObject to enable Basic HTTP Auth. - :param timeout: (optional) Float describing the timeout of the request. - :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. - :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. - :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param **kwargs: Optional arguments that ``request`` takes. """ - return request('PATCH', url, **kwargs) + return request('patch', url, **kwargs) def delete(url, **kwargs): - """Sends a DELETE request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. - :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. - :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. - :param auth: (optional) AuthObject to enable Basic HTTP Auth. - :param timeout: (optional) Float describing the timeout of the request. - :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. - :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param **kwargs: Optional arguments that ``request`` takes. """ - return request('DELETE', url, **kwargs) + return request('delete', url, **kwargs) diff --git a/requests/config.py b/requests/config.py index 39be2ed4..794109c5 100644 --- a/requests/config.py +++ b/requests/config.py @@ -62,6 +62,7 @@ settings.proxies = None settings.verbose = None settings.timeout = None settings.max_redirects = 30 +settings.decode_unicode = True #: Use socket.setdefaulttimeout() as fallback? settings.timeout_fallback = True diff --git a/requests/core.py b/requests/core.py index 655703e5..e44cf83a 100644 --- a/requests/core.py +++ b/requests/core.py @@ -19,9 +19,11 @@ __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' -from models import HTTPError +from models import HTTPError, Request, Response from api import * from exceptions import * from sessions import session from status_codes import codes -from config import settings \ No newline at end of file +from config import settings + +import utils diff --git a/requests/models.py b/requests/models.py index a63c904c..4a304a50 100644 --- a/requests/models.py +++ b/requests/models.py @@ -9,8 +9,10 @@ requests.models import urllib import urllib2 import socket +import codecs import zlib + from urllib2 import HTTPError from urlparse import urlparse, urlunparse, urljoin from datetime import datetime @@ -20,9 +22,9 @@ from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicA from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers -from .utils import dict_from_cookiejar -from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects +from .utils import dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip from .status_codes import codes +from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) @@ -30,7 +32,7 @@ REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) class Request(object): - """The :class:`Request ` object. It carries out all functionality of + """The :class:`Request ` object. It carries out all functionality of Requests. Recommended interface is with the Requests functions. """ @@ -46,24 +48,24 @@ class Request(object): #: Request URL. self.url = url - #: Dictonary of HTTP Headers to attach to the :class:`Request `. + #: Dictonary of HTTP Headers to attach to the :class:`Request `. self.headers = headers #: Dictionary of files to multipart upload (``{filename: content}``). self.files = files - #: HTTP Method to use. Available: GET, HEAD, PUT, POST, DELETE. + #: HTTP Method to use. self.method = method #: Dictionary or byte of request body data to attach to the - #: :class:`Request `. + #: :class:`Request `. self.data = None #: Dictionary or byte of querystring data to attach to the - #: :class:`Request `. + #: :class:`Request `. self.params = None - #: True if :class:`Request ` is part of a redirect chain (disables history + #: True if :class:`Request ` is part of a redirect chain (disables history #: and HTTPError storage). self.redirect = redirect @@ -76,7 +78,7 @@ class Request(object): self.data, self._enc_data = self._encode_params(data) self.params, self._enc_params = self._encode_params(params) - #: :class:`Response ` instance, containing + #: :class:`Response ` instance, containing #: content and metadata of HTTP Response, once :attr:`sent `. self.response = Response() @@ -85,10 +87,10 @@ class Request(object): if not auth: auth = auth_manager.get_auth(self.url) - #: :class:`AuthObject` to attach to :class:`Request `. + #: :class:`AuthObject` to attach to :class:`Request `. self.auth = auth - #: CookieJar to attach to :class:`Request `. + #: CookieJar to attach to :class:`Request `. self.cookiejar = cookiejar #: True if Request has been sent. @@ -134,9 +136,16 @@ class Request(object): _handlers.append(urllib2.HTTPCookieProcessor(self.cookiejar)) if self.auth: - if not isinstance(self.auth.handler, (urllib2.AbstractBasicAuthHandler, urllib2.AbstractDigestAuthHandler)): + if not isinstance(self.auth.handler, + (urllib2.AbstractBasicAuthHandler, + urllib2.AbstractDigestAuthHandler)): + # TODO: REMOVE THIS COMPLETELY - auth_manager.add_password(self.auth.realm, self.url, self.auth.username, self.auth.password) + auth_manager.add_password( + self.auth.realm, self.url, + self.auth.username, + self.auth.password) + self.auth.handler = self.auth.handler(auth_manager) auth_manager.add_auth(self.url, self.auth) @@ -168,7 +177,10 @@ class Request(object): def _build_response(self, resp, is_error=False): - """Build internal :class:`Response ` object from given response.""" + """Build internal :class:`Response ` object + from given response. + """ + def build(resp): @@ -177,9 +189,7 @@ class Request(object): try: response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None)) - response.read = resp.read - response._resp = resp - response._close = resp.close + response.fo = resp if self.cookiejar: response.cookies = dict_from_cookiejar(self.cookiejar) @@ -204,10 +214,15 @@ class Request(object): while ( ('location' in r.headers) and + ((self.method in ('GET', 'HEAD')) or + (r.status_code is codes.see_other) or + (self.allow_redirects)) ((r.status_code is codes.see_other) or (self.allow_redirects)) + ((r.status_code is codes.see_other) or + (self.allow_redirects)) ): - r.close() + r.fo.close() if not len(history) < settings.max_redirects: raise TooManyRedirects() @@ -256,8 +271,8 @@ class Request(object): Otherwise, assumes the data is already encoded appropriately, and returns it twice. - """ + if hasattr(data, 'items'): result = [] for k, vs in data.items(): @@ -301,7 +316,6 @@ class Request(object): """ self._checks() - success = False # Logging if settings.verbose: @@ -367,7 +381,6 @@ class Request(object): self._build_response(why, is_error=True) - else: self._build_response(resp) self.response.ok = True @@ -378,37 +391,46 @@ class Request(object): return self.sent - class Response(object): - """The core :class:`Response ` object. All - :class:`Request ` objects contain a - :class:`response ` attribute, which is an instance + """The core :class:`Response ` object. All + :class:`Request ` objects contain a + :class:`response ` attribute, which is an instance of this class. """ def __init__(self): - #: Raw content of the response, in bytes. - #: If ``content-encoding`` of response was set to ``gzip``, the - #: response data will be automatically deflated. + self._content = None + self._content_consumed = False + #: Integer Code of responded HTTP Status. self.status_code = None + #: Case-insensitive Dictionary of Response Headers. #: For example, ``headers['content-encoding']`` will return the #: value of a ``'Content-Encoding'`` response header. self.headers = CaseInsensitiveDict() + + #: File-like object representation of response (for advanced usage). + self.fo = None + #: Final URL location of Response. self.url = None + #: True if no :attr:`error` occured. self.ok = False + #: Resulting :class:`HTTPError` of request, if one occured. self.error = None - #: A list of :class:`Response ` objects from + + #: A list of :class:`Response ` objects from #: the history of the Request. Any redirect responses will end #: up here. self.history = [] - #: The Request that created the Response. + + #: The :class:`Request ` that created the Response. self.request = None + #: A dictionary of Cookies the server sent back. self.cookies = None @@ -419,23 +441,65 @@ class Response(object): def __nonzero__(self): """Returns true if :attr:`status_code` is 'OK'.""" + return not self.error + def iter_content(self, chunk_size=10 * 1024, decode_unicode=None): + """Iterates over the response data. This avoids reading the content + at once into memory for large responses. The chunk size is the number + of bytes it should read into memory. This is not necessarily the + length of each item returned as decoding can take place. + """ + if self._content_consumed: + raise RuntimeError('The content for this response was ' + 'already consumed') - def __getattr__(self, name): - """Read and returns the full stream when accessing to :attr: `content`""" - if name == 'content': - if self._content is not None: - return self._content - self._content = self.read() - if self.headers.get('content-encoding', '') == 'gzip': - try: - self._content = zlib.decompress(self._content, 16+zlib.MAX_WBITS) - except zlib.error: - pass + def generate(): + while 1: + chunk = self.fo.read(chunk_size) + if not chunk: + break + yield chunk + self._content_consumed = True + gen = generate() + if 'gzip' in self.headers.get('content-encoding', ''): + gen = stream_decode_gzip(gen) + if decode_unicode is None: + decode_unicode = settings.decode_unicode + if decode_unicode: + gen = stream_decode_response_unicode(gen, self) + return gen + + @property + def content(self): + """Content of the response, in bytes or unicode + (if available). + """ + + if self._content is not None: return self._content - else: - raise AttributeError + + if self._content_consumed: + raise RuntimeError('The content for this response was ' + 'already consumed') + + # Read the contents. + self._content = self.fo.read() + + # Decode GZip'd content. + if 'gzip' in self.headers.get('content-encoding', ''): + try: + self._content = decode_gzip(self._content) + except zlib.error: + pass + + # Decode unicode content. + if settings.decode_unicode: + self._content = get_unicode_from_response(self) + + self._content_consumed = True + return self._content + def raise_for_status(self): """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured.""" @@ -443,10 +507,6 @@ class Response(object): raise self.error - def close(self): - if self._resp.fp is not None and hasattr(self._resp.fp, '_sock'): - self._resp.fp._sock.recv = None - self._close() class AuthManager(object): """Requests Authentication Manager.""" diff --git a/requests/utils.py b/requests/utils.py index 8ac78b4e..2e16163b 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -9,15 +9,69 @@ that are also useful for external consumption. """ +import cgi +import codecs import cookielib +import re +import zlib -def dict_from_cookiejar(cookiejar): - """Returns a key/value dictionary from a CookieJar.""" +def header_expand(headers): + """Returns an HTTP Header value string from a dictionary. + + Example expansion:: + + {'text/x-dvi': {'q': '.8', 'mxb': '100000', 'mxt': '5.0'}, 'text/x-c': {}} + # Accept: text/x-dvi; q=.8; mxb=100000; mxt=5.0, text/x-c + + (('text/x-dvi', {'q': '.8', 'mxb': '100000', 'mxt': '5.0'}), ('text/x-c', {})) + # Accept: text/x-dvi; q=.8; mxb=100000; mxt=5.0, text/x-c + """ + + collector = [] + + if isinstance(headers, dict): + headers = headers.items() + + elif isinstance(headers, basestring): + return headers + + for i, (value, params) in enumerate(headers): + + _params = [] + + for (p_k, p_v) in params.items(): + + _params.append('%s=%s' % (p_k, p_v)) + + collector.append(value) + collector.append('; ') + + if len(params): + + collector.append('; '.join(_params)) + + if not len(headers) == i+1: + collector.append(', ') + + + # Remove trailing seperators. + if collector[-1] in (', ', '; '): + del collector[-1] + + return ''.join(collector) + + + +def dict_from_cookiejar(cj): + """Returns a key/value dictionary from a CookieJar. + + :param cj: CookieJar object to extract cookies from. + """ cookie_dict = {} - for _, cookies in cookiejar._cookies.items(): + for _, cookies in cj._cookies.items(): for _, cookies in cookies.items(): for cookie in cookies.values(): # print cookie @@ -27,7 +81,10 @@ def dict_from_cookiejar(cookiejar): def cookiejar_from_dict(cookie_dict): - """Returns a CookieJar from a key/value dictionary.""" + """Returns a CookieJar from a key/value dictionary. + + :param cookie_dict: Dict of key/values to insert into CookieJar. + """ # return cookiejar if one was passed in if isinstance(cookie_dict, cookielib.CookieJar): @@ -42,7 +99,11 @@ def cookiejar_from_dict(cookie_dict): def add_dict_to_cookiejar(cj, cookie_dict): - """Returns a CookieJar from a key/value dictionary.""" + """Returns a CookieJar from a key/value dictionary. + + :param cj: CookieJar to insert cookies into. + :param cookie_dict: Dict of key/values to insert into CookieJar. + """ for k, v in cookie_dict.items(): @@ -70,3 +131,124 @@ def add_dict_to_cookiejar(cj, cookie_dict): cj.set_cookie(cookie) return cj + + +def get_encodings_from_content(content): + """Returns encodings from given content string. + + :param content: bytestring to extract encodings from. + """ + + charset_re = re.compile(r']', flags=re.I) + + return charset_re.findall(content) + + +def get_encoding_from_headers(headers): + """Returns encodings from given HTTP Header Dict. + + :param headers: dictionary to extract encoding from. + """ + + content_type = headers.get('content-type') + + if not content_type: + return None + + content_type, params = cgi.parse_header(content_type) + + if 'charset' in params: + return params['charset'].strip("'\"") + + +def unicode_from_html(content): + """Attempts to decode an HTML string into unicode. + If unsuccessful, the original content is returned. + """ + + encodings = get_encodings_from_content(content) + + for encoding in encodings: + + try: + return unicode(content, encoding) + except (UnicodeError, TypeError): + pass + + return content + + +def stream_decode_response_unicode(iterator, r): + """Stream decodes a iterator.""" + encoding = get_encoding_from_headers(r.headers) + if encoding is None: + for item in iterator: + yield item + return + + decoder = codecs.getincrementaldecoder(encoding)(errors='replace') + for chunk in iterator: + rv = decoder.decode(chunk) + if rv: + yield rv + rv = decoder.decode('', final=True) + if rv: + yield rv + + +def get_unicode_from_response(r): + """Returns the requested content back in unicode. + + :param r: Reponse object to get unicode content from. + + Tried: + + 1. charset from content-type + + 2. every encodings from ```` + + 3. fall back and replace all unicode characters + + """ + + tried_encodings = [] + + # Try charset from content-type + encoding = get_encoding_from_headers(r.headers) + + if encoding: + try: + return unicode(r.content, encoding) + except UnicodeError: + tried_encodings.append(encoding) + + # Fall back: + try: + return unicode(r.content, encoding, errors='replace') + except TypeError: + return r.content + + +def decode_gzip(content): + """Return gzip-decoded string. + + :param content: bytestring to gzip-decode. + """ + + return zlib.decompress(content, 16 + zlib.MAX_WBITS) + + +def stream_decode_gzip(iterator): + """Stream decodes a gzip-encoded iterator""" + try: + dec = zlib.decompressobj(16 + zlib.MAX_WBITS) + for chunk in iterator: + rv = dec.decompress(chunk) + if rv: + yield rv + buf = dec.decompress('') + rv = buf + dec.flush() + if rv: + yield rv + except zlib.error: + pass diff --git a/test_requests.py b/test_requests.py index 659bd5ae..83b827ab 100755 --- a/test_requests.py +++ b/test_requests.py @@ -62,6 +62,13 @@ class RequestsTestSuite(unittest.TestCase): r = requests.get(httpbin('/')) self.assertEqual(r.status_code, 200) + def test_HTTP_302_ALLOW_REDIRECT_GET(self): + r = requests.get(httpbin('redirect', '1')) + self.assertEqual(r.status_code, 200) + + def test_HTTP_302_GET(self): + r = requests.get(httpbin('redirect', '1'), allow_redirects=False) + self.assertEqual(r.status_code, 302) def test_HTTPS_200_OK_GET(self): r = requests.get(httpsbin('/')) @@ -490,6 +497,5 @@ class RequestsTestSuite(unittest.TestCase): self.assertEqual(r2.status_code, 200) - if __name__ == '__main__': unittest.main()