diff --git a/AUTHORS b/AUTHORS index 62a2414d..e09b3668 100644 --- a/AUTHORS +++ b/AUTHORS @@ -17,3 +17,4 @@ Patches and Suggestions - Justin Murphy - Rob Madole - Aram Dulyan +- Johannes Gorset diff --git a/HISTORY.rst b/HISTORY.rst index d1dc02c7..0eaf42ab 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,14 @@ History ------- +0.3.3 (2011-05-12) +++++++++++++++++++ + +* Request timeouts +* Unicode url-encoded data +* Settings context manager and module + + 0.3.2 (2011-04-15) ++++++++++++++++++ diff --git a/README.rst b/README.rst index 3369a850..0546a121 100644 --- a/README.rst +++ b/README.rst @@ -63,24 +63,24 @@ If a {filename: fileobject} dictionary is passed in (files=...), a multipart_enc If CookieJar object is is passed in (cookies=...), the cookies will be sent with the request. GET Requests - >>> request.get(url, params={}, headers={}, cookies=None, auth=None) - + >>> requests.get(url, params={}, headers={}, cookies=None, auth=None) + HEAD Requests - >>> request.head(url, params={}, headers={}, cookies=None, auth=None) - + >>> requests.head(url, params={}, headers={}, cookies=None, auth=None) + PUT Requests - >>> request.put(url, data='', headers={}, files={}, cookies=None, auth=None) - + >>> requests.put(url, data='', headers={}, files={}, cookies=None, auth=None) + POST Requests - >>> request.post(url, data={}, headers={}, files={}, cookies=None, auth=None) - + >>> requests.post(url, data={}, headers={}, files={}, cookies=None, auth=None) + DELETE Requests - >>> request.delete(url, params={}, headers={}, cookies=None, auth=None) - + >>> requests.delete(url, params={}, headers={}, cookies=None, auth=None) + **Responses:** diff --git a/docs/conf.py b/docs/conf.py index 8ea77310..5da948ba 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,7 +25,7 @@ import sys, os # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.pngmath', 'sphinx.ext.jsmath', 'sphinx.ext.viewcode'] +extensions = ['sphinx.ext.autodoc'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -48,7 +48,7 @@ copyright = u'2011, Kenneth Reitz' # built documents. # # The short X.Y version. -version = '0.3.2' +version = '0.3.3' # The full version, including alpha/beta/rc tags. release = version diff --git a/requests/__init__.py b/requests/__init__.py index d44b2667..cf171f50 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -1,6 +1,31 @@ # -*- coding: utf-8 -*- +import inspect + import packages from core import * from core import __version__ + +timeout = None + +class settings: + """Context manager for settings.""" + + cache = {} + + def __init__(self, timeout): + self.module = inspect.getmodule(self) + + # Cache settings + self.cache['timeout'] = self.module.timeout + + self.module.timeout = timeout + + def __enter__(self): + pass + + def __exit__(self, type, value, traceback): + # Restore settings + for key in self.cache: + setattr(self.module, key, self.cache[key]) diff --git a/requests/core.py b/requests/core.py index 625d845f..bb62fcee 100644 --- a/requests/core.py +++ b/requests/core.py @@ -12,8 +12,10 @@ from __future__ import absolute_import +import requests import urllib import urllib2 +import socket import zlib from urllib2 import HTTPError @@ -23,10 +25,9 @@ from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers - __title__ = 'requests' -__version__ = '0.3.2' -__build__ = 0x000302 +__version__ = '0.3.3' +__build__ = 0x000303 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' @@ -38,7 +39,6 @@ __all__ = [ ] - class _Request(urllib2.Request): """Hidden wrapper around the urllib2.Request object. Allows for manual setting of HTTP methods. @@ -63,7 +63,7 @@ class Request(object): _METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE') def __init__(self, url=None, headers=dict(), files=None, method=None, - data=dict(), auth=None, cookiejar=None): + data=dict(), auth=None, cookiejar=None, timeout=None): self.url = url self.headers = headers @@ -71,11 +71,16 @@ class Request(object): self.method = method self.data = data + socket.setdefaulttimeout(timeout) + + for (k, v) in self.data.iteritems(): + self.data[k] = v.encode('utf-8') + # url encode data if it's a dict if hasattr(data, 'items'): - self._enc_data = urllib.urlencode(data) + self._enc_data = urllib.urlencode(self.data) else: - self._enc_data = data + self._enc_data = self.data self.response = Response() @@ -155,8 +160,9 @@ class Request(object): self.response.url = getattr(resp, 'url', None) + @staticmethod - def _build_url(url, data): + def _build_url(url, data=None): """Build URLs.""" if urlparse(url).query: @@ -167,6 +173,7 @@ class Request(object): else: return url + def send(self, anyway=False): """Sends the request. Returns True of successful, false if not. If there was an HTTPError during transmission, @@ -200,6 +207,9 @@ class Request(object): req.headers.update(self.headers) if not self.sent or anyway: + + + try: opener = self._get_opener() resp = opener(req) @@ -226,6 +236,8 @@ class Request(object): def read(self, *args): return self.response.read() + + class Response(object): """The :class:`Request` object. All :class:`Request` objects contain a :class:`Request.response ` attribute, which is an instance of @@ -260,6 +272,7 @@ class Response(object): return self.content + class AuthManager(object): """Authentication Manager.""" @@ -296,6 +309,7 @@ class AuthManager(object): self._auth[uri] = auth + def add_password(self, realm, uri, user, passwd): """Adds password to AuthManager.""" # uri could be a single URI or a sequence @@ -443,18 +457,20 @@ def request(method, url, **kwargs): :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. """ data = kwargs.pop('data', dict()) or kwargs.pop('params', dict()) r = Request(method=method, url=url, data=data, headers=kwargs.pop('headers', {}), cookiejar=kwargs.pop('cookies', None), files=kwargs.pop('files', None), - auth=kwargs.pop('auth', auth_manager.get_auth(url))) + auth=kwargs.pop('auth', auth_manager.get_auth(url)), + timeout=kwargs.pop('timeout', requests.timeout)) r.send() return r.response -def get(url, params={}, headers={}, cookies=None, auth=None): +def get(url, params={}, headers={}, cookies=None, auth=None, **kwargs): """Sends a GET request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. @@ -462,12 +478,13 @@ def get(url, params={}, headers={}, cookies=None, auth=None): :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. """ - return request('GET', url, params=params, headers=headers, cookies=cookies, auth=auth) + return request('GET', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs) -def head(url, params={}, headers={}, cookies=None, auth=None): +def head(url, params={}, headers={}, cookies=None, auth=None, **kwargs): """Sends a HEAD request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. @@ -475,12 +492,13 @@ def head(url, params={}, headers={}, cookies=None, auth=None): :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. """ - return request('HEAD', url, params=params, headers=headers, cookies=cookies, auth=auth) + return request('HEAD', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs) -def post(url, data={}, headers={}, files=None, cookies=None, auth=None): +def post(url, data={}, headers={}, files=None, cookies=None, auth=None, **kwargs): """Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. @@ -489,12 +507,13 @@ def post(url, data={}, headers={}, files=None, cookies=None, auth=None): :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. """ - return request('POST', url, data=data, headers=headers, files=files, cookies=cookies, auth=auth) + return request('POST', url, data=data, headers=headers, files=files, cookies=cookies, auth=auth, **kwargs) -def put(url, data='', headers={}, files={}, cookies=None, auth=None): +def put(url, data='', headers={}, files={}, cookies=None, auth=None, **kwargs): """Sends a PUT request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. @@ -503,12 +522,13 @@ def put(url, data='', headers={}, files={}, cookies=None, auth=None): :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. """ - return request('PUT', url, data=data, headers=headers, files=files, cookies=cookies, auth=auth) + return request('PUT', url, data=data, headers=headers, files=files, cookies=cookies, auth=auth, **kwargs) -def delete(url, params={}, headers={}, cookies=None, auth=None): +def delete(url, params={}, headers={}, cookies=None, auth=None, **kwargs): """Sends a DELETE request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. @@ -516,9 +536,10 @@ def delete(url, params={}, headers={}, cookies=None, auth=None): :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. """ - return request('DELETE', url, params=params, headers=headers, cookies=cookies, auth=auth) + return request('DELETE', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs) diff --git a/requests/structures.py b/requests/structures.py new file mode 100644 index 00000000..96b4ea4e --- /dev/null +++ b/requests/structures.py @@ -0,0 +1,360 @@ + +# from: werkzeug + +class TypeConversionDict(dict): + """Works like a regular dict but the :meth:`get` method can perform + type conversions. :class:`MultiDict` and :class:`CombinedMultiDict` + are subclasses of this class and provide the same feature. + """ + + def get(self, key, default=None, type=None): + """Return the default value if the requested data doesn't exist. + If `type` is provided and is a callable it should convert the value, + return it or raise a :exc:`ValueError` if that is not possible. In + this case the function will return the default as if the value was not + found: + + >>> d = TypeConversionDict(foo='42', bar='blub') + >>> d.get('foo', type=int) + 42 + >>> d.get('bar', -1, type=int) + -1 + + :param key: The key to be looked up. + :param default: The default value to be returned if the key can't + be looked up. If not further specified `None` is + returned. + :param type: A callable that is used to cast the value in the + :class:`MultiDict`. If a :exc:`ValueError` is raised + by this callable the default value is returned. + """ + try: + rv = self[key] + if type is not None: + rv = type(rv) + except (KeyError, ValueError): + rv = default + return rv + + + +# from: werkzeug + +class MultiDict(TypeConversionDict): + """A :class:`MultiDict` is a dictionary subclass customized to deal with + multiple values for the same key which is for example used by the parsing + functions in the wrappers. This is necessary because some HTML form + elements pass multiple values for the same key. + + :class:`MultiDict` implements all standard dictionary methods. + Internally, it saves all values for a key as a list, but the standard dict + access methods will only return the first value for a key. If you want to + gain access to the other values, too, you have to use the `list` methods as + explained below. + + Basic Usage: + + >>> d = MultiDict([('a', 'b'), ('a', 'c')]) + >>> d + MultiDict([('a', 'b'), ('a', 'c')]) + >>> d['a'] + 'b' + >>> d.getlist('a') + ['b', 'c'] + >>> 'a' in d + True + + It behaves like a normal dict thus all dict functions will only return the + first value when multiple values for one key are found. + + From Werkzeug 0.3 onwards, the `KeyError` raised by this class is also a + subclass of the :exc:`~exceptions.BadRequest` HTTP exception and will + render a page for a ``400 BAD REQUEST`` if caught in a catch-all for HTTP + exceptions. + + A :class:`MultiDict` can be constructed from an iterable of + ``(key, value)`` tuples, a dict, a :class:`MultiDict` or from Werkzeug 0.2 + onwards some keyword parameters. + + :param mapping: the initial value for the :class:`MultiDict`. Either a + regular dict, an iterable of ``(key, value)`` tuples + or `None`. + """ + + # the key error this class raises. Because of circular dependencies + # with the http exception module this class is created at the end of + # this module. + KeyError = None + + def __init__(self, mapping=None): + if isinstance(mapping, MultiDict): + dict.__init__(self, ((k, l[:]) for k, l in mapping.iterlists())) + elif isinstance(mapping, dict): + tmp = {} + for key, value in mapping.iteritems(): + if isinstance(value, (tuple, list)): + value = list(value) + else: + value = [value] + tmp[key] = value + dict.__init__(self, tmp) + else: + tmp = {} + for key, value in mapping or (): + tmp.setdefault(key, []).append(value) + dict.__init__(self, tmp) + + def __getstate__(self): + return dict(self.lists()) + + def __setstate__(self, value): + dict.clear(self) + dict.update(self, value) + + def __iter__(self): + return self.iterkeys() + + def __getitem__(self, key): + """Return the first data value for this key; + raises KeyError if not found. + + :param key: The key to be looked up. + :raise KeyError: if the key does not exist. + """ + if key in self: + return dict.__getitem__(self, key)[0] + raise self.KeyError(key) + + def __setitem__(self, key, value): + """Like :meth:`add` but removes an existing key first. + + :param key: the key for the value. + :param value: the value to set. + """ + dict.__setitem__(self, key, [value]) + + def add(self, key, value): + """Adds a new value for the key. + + .. versionadded:: 0.6 + + :param key: the key for the value. + :param value: the value to add. + """ + dict.setdefault(self, key, []).append(value) + + def getlist(self, key, type=None): + """Return the list of items for a given key. If that key is not in the + `MultiDict`, the return value will be an empty list. Just as `get` + `getlist` accepts a `type` parameter. All items will be converted + with the callable defined there. + + :param key: The key to be looked up. + :param type: A callable that is used to cast the value in the + :class:`MultiDict`. If a :exc:`ValueError` is raised + by this callable the value will be removed from the list. + :return: a :class:`list` of all the values for the key. + """ + try: + rv = dict.__getitem__(self, key) + except KeyError: + return [] + if type is None: + return list(rv) + result = [] + for item in rv: + try: + result.append(type(item)) + except ValueError: + pass + return result + + def setlist(self, key, new_list): + """Remove the old values for a key and add new ones. Note that the list + you pass the values in will be shallow-copied before it is inserted in + the dictionary. + + >>> d = MultiDict() + >>> d.setlist('foo', ['1', '2']) + >>> d['foo'] + '1' + >>> d.getlist('foo') + ['1', '2'] + + :param key: The key for which the values are set. + :param new_list: An iterable with the new values for the key. Old values + are removed first. + """ + dict.__setitem__(self, key, list(new_list)) + + def setdefault(self, key, default=None): + """Returns the value for the key if it is in the dict, otherwise it + returns `default` and sets that value for `key`. + + :param key: The key to be looked up. + :param default: The default value to be returned if the key is not + in the dict. If not further specified it's `None`. + """ + if key not in self: + self[key] = default + else: + default = self[key] + return default + + def setlistdefault(self, key, default_list=None): + """Like `setdefault` but sets multiple values. The list returned + is not a copy, but the list that is actually used internally. This + means that you can put new values into the dict by appending items + to the list: + + >>> d = MultiDict({"foo": 1}) + >>> d.setlistdefault("foo").extend([2, 3]) + >>> d.getlist("foo") + [1, 2, 3] + + :param key: The key to be looked up. + :param default: An iterable of default values. It is either copied + (in case it was a list) or converted into a list + before returned. + :return: a :class:`list` + """ + if key not in self: + default_list = list(default_list or ()) + dict.__setitem__(self, key, default_list) + else: + default_list = dict.__getitem__(self, key) + return default_list + + def items(self, multi=False): + """Return a list of ``(key, value)`` pairs. + + :param multi: If set to `True` the list returned will have a + pair for each value of each key. Otherwise it + will only contain pairs for the first value of + each key. + + :return: a :class:`list` + """ + return list(self.iteritems(multi)) + + def lists(self): + """Return a list of ``(key, values)`` pairs, where values is the list of + all values associated with the key. + + :return: a :class:`list` + """ + return list(self.iterlists()) + + def values(self): + """Returns a list of the first value on every key's value list. + + :return: a :class:`list`. + """ + return [self[key] for key in self.iterkeys()] + + def listvalues(self): + """Return a list of all values associated with a key. Zipping + :meth:`keys` and this is the same as calling :meth:`lists`: + + >>> d = MultiDict({"foo": [1, 2, 3]}) + >>> zip(d.keys(), d.listvalues()) == d.lists() + True + + :return: a :class:`list` + """ + return list(self.iterlistvalues()) + + def iteritems(self, multi=False): + """Like :meth:`items` but returns an iterator.""" + for key, values in dict.iteritems(self): + if multi: + for value in values: + yield key, value + else: + yield key, values[0] + + def iterlists(self): + """Like :meth:`items` but returns an iterator.""" + for key, values in dict.iteritems(self): + yield key, list(values) + + def itervalues(self): + """Like :meth:`values` but returns an iterator.""" + for values in dict.itervalues(self): + yield values[0] + + def iterlistvalues(self): + """Like :meth:`listvalues` but returns an iterator.""" + return dict.itervalues(self) + + def copy(self): + """Return a shallow copy of this object.""" + return self.__class__(self) + + def to_dict(self, flat=True): + """Return the contents as regular dict. If `flat` is `True` the + returned dict will only have the first item present, if `flat` is + `False` all values will be returned as lists. + + :param flat: If set to `False` the dict returned will have lists + with all the values in it. Otherwise it will only + contain the first value for each key. + :return: a :class:`dict` + """ + if flat: + return dict(self.iteritems()) + return dict(self.lists()) + + def update(self, other_dict): + """update() extends rather than replaces existing key lists.""" + for key, value in iter_multi_items(other_dict): + MultiDict.add(self, key, value) + + def pop(self, key, default=_missing): + """Pop the first item for a list on the dict. Afterwards the + key is removed from the dict, so additional values are discarded: + + >>> d = MultiDict({"foo": [1, 2, 3]}) + >>> d.pop("foo") + 1 + >>> "foo" in d + False + + :param key: the key to pop. + :param default: if provided the value to return if the key was + not in the dictionary. + """ + try: + return dict.pop(self, key)[0] + except KeyError, e: + if default is not _missing: + return default + raise self.KeyError(str(e)) + + def popitem(self): + """Pop an item from the dict.""" + try: + item = dict.popitem(self) + return (item[0], item[1][0]) + except KeyError, e: + raise self.KeyError(str(e)) + + def poplist(self, key): + """Pop the list for a key from the dict. If the key is not in the dict + an empty list is returned. + + .. versionchanged:: 0.5 + If the key does no longer exist a list is returned instead of + raising an error. + """ + return dict.pop(self, key, []) + + def popitemlist(self): + """Pop a ``(key, list)`` tuple from the dict.""" + try: + return dict.popitem(self) + except KeyError, e: + raise self.KeyError(str(e)) + + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, self.items(multi=True)) diff --git a/test_requests.py b/test_requests.py index 9b3dae9c..c73803c0 100755 --- a/test_requests.py +++ b/test_requests.py @@ -139,5 +139,11 @@ class RequestsTestSuite(unittest.TestCase): r = requests.get('https://convore.com/api/account/verify.json') self.assertEquals(r.status_code, 200) + def test_unicode_get(self): + requests.get('http://google.com', params={'foo': u'føø'}) + requests.get('http://google.com', params={'foo': u'foo'}) + requests.get('http://google.com/ø', params={'foo': u'foo'}) + + if __name__ == '__main__': unittest.main()