diff --git a/requests/api.py b/requests/api.py index dea4c1ad..6b1839a7 100644 --- a/requests/api.py +++ b/requests/api.py @@ -14,7 +14,7 @@ This module implements the Requests API. from ._config import get_config from .models import Request, Response from .status_codes import codes -from .hooks import dispatch_hook +from hooks import setup_hooks, dispatch_hooks from .utils import cookiejar_from_dict, header_expand @@ -46,8 +46,9 @@ def request(method, url, method = str(method).upper() config = get_config(config) - if cookies is None: - cookies = {} + cookies = cookiejar_from_dict(cookies if cookies is not None else dict()) + + cookies = cookiejar_from_dict(cookies) # cookies = cookiejar_from_dict(cookies) @@ -71,15 +72,17 @@ def request(method, url, proxies=proxies or config.get('proxies'), _pools=_pools ) + + hooks = setup_hooks(hooks if hooks is not None else dict()) # Arguments manipulation hook. - args = dispatch_hook('args', hooks, args) + args = dispatch_hooks(hooks['args'], args) # Create Request object. r = Request(**args) # Pre-request hook. - r = dispatch_hook('pre_request', hooks, r) + r = dispatch_hooks(hooks['pre_request'], r) # Only construct the request (for async) if _return_request: @@ -90,10 +93,10 @@ def request(method, url, # TODO: Add these hooks inline. # Post-request hook. - r = dispatch_hook('post_request', hooks, r) + r = dispatch_hooks(hooks['post_request'], r) # Response manipulation hook. - r.response = dispatch_hook('response', hooks, r.response) + r.response = dispatch_hooks(hooks['response'], r.response) return r.response diff --git a/requests/config.py b/requests/config.py new file mode 100644 index 00000000..e6adfca6 --- /dev/null +++ b/requests/config.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- + +""" +requests.config +~~~~~~~~~~~~~~~ + +This module provides the Requests settings feature set. + +""" + +class Settings(object): + _singleton = {} + + # attributes with defaults + __attrs__ = [] + + def __init__(self, **kwargs): + super(Settings, self).__init__() + + self.__dict__ = self._singleton + + + def __call__(self, *args, **kwargs): + # new instance of class to call + r = self.__class__() + + # cache previous settings for __exit__ + r.__cache = self.__dict__.copy() + map(self.__cache.setdefault, self.__attrs__) + + # set new settings + self.__dict__.update(*args, **kwargs) + + return r + + + def __enter__(self): + pass + + + def __exit__(self, *args): + + # restore cached copy + self.__dict__.update(self.__cache.copy()) + del self.__cache + + + def __getattribute__(self, key): + if key in object.__getattribute__(self, '__attrs__'): + try: + return object.__getattribute__(self, key) + except AttributeError: + return None + return object.__getattribute__(self, key) + + +settings = Settings() + +settings.base_headers = { + 'User-Agent': 'python-requests.org', + 'Accept-Encoding': ', '.join([ 'identity', 'deflate', 'compress', 'gzip' ]), +} +settings.accept_gzip = True +settings.proxies = None +settings.verbose = None +settings.timeout = None +settings.max_redirects = 30 +settings.decode_unicode = False +settings.gracefull_hooks = True + +#: A dictionary of default hooks to be applied, based on settings. +settings.default_hooks = { + 'args': list(), + 'pre_request': list(), + 'post_request': list(), + 'response': list() +} + +#: Use socket.setdefaulttimeout() as fallback? +settings.timeout_fallback = True diff --git a/requests/hooks.py b/requests/hooks.py index 2938029b..e2980ac3 100644 --- a/requests/hooks.py +++ b/requests/hooks.py @@ -23,18 +23,162 @@ Available hooks: """ import warnings +from collections import Iterable +import config +import zlib +import bz2 +from cgi import parse_header +def setup_hooks(supplied): + """Setup a hooks mapping, based on the supplied argument. Eache mapping + value will be list of hooks that will extend the **default_hooks**. -def dispatch_hook(key, hooks, hook_data): - """Dipatches a hook dictionary on a given peice of data.""" + :param supplied: a dictionary of hooks. Each value can either be a callable + or a list of callables. + :type supplied: dict + :returns: a dictionary of hooks that extends the **default_hooks** dictionary. + :rtype: dict + """ - hooks = hooks or dict() + # Copy the default hooks settings. + default = config.settings.default_hooks + dispatching = dict([(k, v[:]) for k, v in default.items()]) - if key in hooks: + # I abandoned the idea of a dictionary of sets because sets may not keep + # insertion order, while it may be important. Also, there is no real reason + # to force hooks to run once. + for hooks, values in supplied.items(): + hook_list = values if isinstance(values, Iterable) else [values] + dispatching[hooks].extend(hook_list) + + # If header is set by config, maybe response is encoded. + if config.settings.base_headers.get('Accept-Encoding', ''): + if not decode_encoding in dispatching['response']: + # It's safer to put decoding as first hook. + dispatching['response'].insert(0, decode_encoding) + + if config.settings.decode_unicode: try: - return hooks.get(key).__call__(hook_data) or hook_data + # Try unicode encoding just after content decoding... + index = dispatching['response'].index(decode_encoding) + 1 + except ValueError: + # ... Or as first hook + index = 0 + dispatching['response'].insert(index, decode_unicode) + + return dispatching + +def dispatch_hooks(hooks, data): + """Dispatches multiple hooks on a given piece of data. + + :param key: the hooks group to lookup + :type key: str + :param hooks: the hooks dictionary. The value of each key can be a callable + object, or a list of callable objects. + :type hooks: dict + :param data: the object on witch the hooks should be applied + :type data: object + """ + for hook in hooks: + try: + # hook must be a callable. + data = hook(data) except Exception, why: + + # Letting users to choose a policy may be an idea. It can be as + # simple as "be gracefull, or not": + # + # config.settings.gracefull_hooks = True | False + if not config.settings.gracefull_hooks: raise + warnings.warn(str(why)) - return hook_data + return data + +#: Example response hook that turns a JSON formatted +#: :py:class:`requests.models.Response.content` into a dumped data structure:: +#: +#: try: +#: import json +#: except ImportError: +#: try: +#: import simplejson as json +#: except ImportError: +#: json = False +#: +#: if json: +#: def json_content(r): +#: """Turns content into a dumped JSON structure.""" +#: r._content = json.dumps(r.content) +#: return r +#: +#: Example response hook that turns an XML formatted +#: :py:class:`requests.models.Response.content` into an ElementTree:: +#: +#: try: +#: from lxml import etree +#: except ImportError: +#: try: +#: import xml.etree.cElementTree as etree +#: except ImportError: +#: try: +#: import xml.etree.ElementTree as etree +#: except ImportError: +#: try: +#: import cElementTree as etree +#: except ImportError: +#: try: +#: import elementtree.ElementTree as etree +#: except ImportError: +#: etree = False +#: +#: if etree: +#: def etree_content(r): +#: """Turns content into an ElementTree structure.""" +#: r._content = etree.fromstring(r.content) +#: return r + +def decode_unicode(r): + """Encode content into unicode string. + + :param r: response object + :type r: :py:class:`requests.models.Response` + :returns: the same input object. + :rtype: :py:class:`requests.models.Response` + """ + content_type, params = parse_header(r.headers['content-type']) + charset = params.get('charset', '').strip("'\"") + r._content = unicode(r.content, charset) if charset else unicode(r.content) + return r + +def decode_encoding(r): + """Decode content using Contetn-Encoding header. + + :param r: response object + :type r: :py:class:`requests.models.Response` + :returns: the same input object. + :rtype: :py:class:`requests.models.Response` + """ + + # Dictionary of content decoders. + decode = { + # No decoding applied. + 'identity': lambda content: content, + # Decode Response content compressed with deflate. + 'deflate': lambda content: zlib.decompress(content), + # Decode Response content compressed with gzip. + 'gzip': lambda content: zlib.decompress(content, 16+zlib.MAX_WBITS), + # Decode Response content compressed with bz2. + # Not a standard Content-Encoding value, but.. + 'bzip2': lambda content: bz2.decompress(content), + } + # Decode Response content compressed with compress. + # If I understood zlib... + decode['compress'] = decode['deflate'] + + # Apply decoding only if the header is set. + encoding = r.headers['content-encoding'] + if encoding: + r._content = decode[encoding](r.content) + return r diff --git a/requests/models.py b/requests/models.py index 27cae2a5..2c863b88 100644 --- a/requests/models.py +++ b/requests/models.py @@ -452,14 +452,21 @@ class Response(object): (if available). """ - if self._content is not None: - return self._content + if self._content is None: + # Read the contents. + self._content = self.fo.read() if self._content_consumed: raise RuntimeError( 'The content for this response was already consumed') + # # Decode GZip'd content. + # if 'gzip' in self.headers.get('content-encoding', ''): + # try: + # self._content = decode_gzip(self._content) + # except zlib.error: + # pass - # Read the contents. + self._content = self.fo.read() self._content = self.raw.read() or self.raw.data # Decode GZip'd content. @@ -469,9 +476,22 @@ class Response(object): except zlib.error: pass + # Decode unicode content. + if settings.decode_unicode: + self._content = get_unicode_from_response(self) + # Decode GZip'd content. + if 'gzip' in self.headers.get('content-encoding', ''): + try: + self._content = decode_gzip(self._content) + except zlib.error: + pass + # Decode unicode content. if self.config.get('decode_unicode'): self._content = get_unicode_from_response(self) + # # Decode unicode content. + # if settings.decode_unicode: + # self._content = get_unicode_from_response(self) self._content_consumed = True return self._content