diff --git a/.gitignore b/.gitignore index f793004e..49e07472 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ nosetests.xml pylint.txt *.pyc docs/_build -toy.py \ No newline at end of file +toy.py +.gitignore diff --git a/AUTHORS b/AUTHORS index 1f7c743f..b6e45cf4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -28,3 +28,6 @@ Patches and Suggestions - Alberto Paro - Jérémy Bethmont - 潘旭 (Xu Pan) +- Tamás Gulácsi +- Rubén Abad +- Peter Manser \ No newline at end of file diff --git a/HISTORY.rst b/HISTORY.rst index 6b60210f..171dae44 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,10 +1,17 @@ History ------- -0.5.1 (?) -+++++++++ +0.5.1 (2011-07-23) +++++++++++++++++++ * International Domain Name Support! +* Access headers without fetching entire body (``read()``) +* Use lists as dicts for parameters +* Add Forced Basic Authentication +* Forced Basic is default authentication type +* ``python-requests.org`` default User-Agent header +* CaseInsensitiveDict lower-case caching +* Response.history bugfix 0.5.0 (2011-06-21) diff --git a/README.rst b/README.rst index f4b37146..96a1f537 100644 --- a/README.rst +++ b/README.rst @@ -43,7 +43,7 @@ HTTPS? Basic Authentication? :: Uh oh, we're not authorized! Let's add authentication. :: - >>> r = requests.get(https://httpbin.ep.io/basic-auth/user/pass', auth=('user', 'pass')) + >>> r = requests.get('https://httpbin.ep.io/basic-auth/user/pass', auth=('user', 'pass')) >>> r.status_code 200 diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 00000000..519752c4 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +python-requests (0.4.1-0) testing; urgency=low + + * Initial Debian package + + -- Bruno Clermont Thu, 26 May 2011 16:25:00 -0500 diff --git a/debian/compat b/debian/compat new file mode 100644 index 00000000..7ed6ff82 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +5 diff --git a/debian/control b/debian/control new file mode 100644 index 00000000..5c6ce49a --- /dev/null +++ b/debian/control @@ -0,0 +1,13 @@ +Source: python-requests +Section: python +Priority: optional +Maintainer: Bruno Clermont +Homepage: https://github.com/bclermont/requests +Bugs: https://github.com/bclermont/requests/issues +Build-Depends: debhelper, python-support + +Package: python-requests +Architecture: all +Depends: ${python:Depends}, python-support +Provides: ${python:Provides} +Description: Python HTTP Requests for Humans. diff --git a/debian/docs b/debian/docs new file mode 100644 index 00000000..9bb74d2d --- /dev/null +++ b/debian/docs @@ -0,0 +1 @@ +docs/user/*.rst \ No newline at end of file diff --git a/debian/pyversions b/debian/pyversions new file mode 100644 index 00000000..8b253bc3 --- /dev/null +++ b/debian/pyversions @@ -0,0 +1 @@ +2.4- diff --git a/debian/rules b/debian/rules new file mode 100755 index 00000000..28e92ec0 --- /dev/null +++ b/debian/rules @@ -0,0 +1,42 @@ +#!/usr/bin/make -f + +# Verbose mode +#export DH_VERBOSE=1 + +clean: + dh_testdir + dh_testroot + + rm -rf build requests.egg-info +# find django-sentry/ -name *.pyc | xargs rm -f + + dh_clean + +build: + dh_testdir + + python setup.py build + +install: + dh_testdir + dh_installdirs + + python setup.py install --root $(CURDIR)/debian/python-requests + +binary-indep: install + +binary-arch: install + dh_install + dh_installdocs +# dh_installchangelogs + dh_compress + dh_fixperms + dh_pysupport + dh_gencontrol + dh_installdeb + dh_md5sums + dh_builddeb -- -Z lzma -z9 + +binary: binary-indep binary-arch +.PHONY: build clean binary-indep binary-arch binary + diff --git a/requests/config.py b/requests/config.py index 0878da92..1d335f42 100644 --- a/requests/config.py +++ b/requests/config.py @@ -53,4 +53,6 @@ class Settings(object): return None return object.__getattribute__(self, key) -settings = Settings() \ No newline at end of file +settings = Settings() +settings.base_headers = {'User-Agent': 'python-requests.org'} +settings.accept_gzip = True \ No newline at end of file diff --git a/requests/core.py b/requests/core.py index 87f55e45..5215fca2 100644 --- a/requests/core.py +++ b/requests/core.py @@ -12,8 +12,8 @@ This module implements the main Requests system. """ __title__ = 'requests' -__version__ = '0.5.0' -__build__ = 0x000500 +__version__ = '0.5.1' +__build__ = 0x000501 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' diff --git a/requests/models.py b/requests/models.py index bb15dc9b..619d099f 100644 --- a/requests/models.py +++ b/requests/models.py @@ -16,7 +16,7 @@ from urlparse import urlparse, urlunparse from datetime import datetime from .config import settings -from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler +from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers @@ -81,6 +81,23 @@ class Request(object): self.sent = False + # Header manipulation and defaults. + + if settings.accept_gzip: + settings.base_headers.update({'Accept-Encoding': 'gzip'}) + + if headers: + headers = CaseInsensitiveDict(self.headers) + else: + headers = CaseInsensitiveDict() + + for (k, v) in settings.base_headers.items(): + if k not in headers: + headers[k] = v + + self.headers = headers + + def __repr__(self): return '' % (self.method) @@ -165,10 +182,7 @@ class Request(object): r = build(resp) - if r.status_code in REDIRECT_STATI: - self.redirect = True - - if self.redirect: + if r.status_code in REDIRECT_STATI and not self.redirect: while ( ('location' in r.headers) and @@ -211,18 +225,19 @@ class Request(object): """Encode parameters in a piece of data. If the data supplied is a dictionary, encodes each parameter in it, and - returns the dictionary of encoded parameters, and a urlencoded version - of that. + returns a list of tuples containing the encoded parameters, and a urlencoded + version of that. Otherwise, assumes the data is already encoded appropriately, and returns it twice. """ if hasattr(data, 'items'): - result = {} - for (k, v) in data.items(): - result[k.encode('utf-8') if isinstance(k, unicode) else k] \ - = v.encode('utf-8') if isinstance(v, unicode) else v + result = [] + for k, vs in data.items(): + for v in isinstance(vs, list) and vs or [vs]: + result.append((k.encode('utf-8') if isinstance(k, unicode) else k, + v.encode('utf-8') if isinstance(v, unicode) else v)) return result, urllib.urlencode(result, doseq=True) else: return data, data @@ -535,17 +550,18 @@ class AuthObject(object): _handlers = { 'basic': HTTPBasicAuthHandler, + 'forced_basic': HTTPForcedBasicAuthHandler, 'digest': HTTPDigestAuthHandler, 'proxy_basic': urllib2.ProxyBasicAuthHandler, 'proxy_digest': urllib2.ProxyDigestAuthHandler } - def __init__(self, username, password, handler='basic', realm=None): + def __init__(self, username, password, handler='forced_basic', realm=None): self.username = username self.password = password self.realm = realm if isinstance(handler, basestring): - self.handler = self._handlers.get(handler.lower(), urllib2.HTTPBasicAuthHandler) + self.handler = self._handlers.get(handler.lower(), HTTPForcedBasicAuthHandler) else: self.handler = handler diff --git a/requests/monkeys.py b/requests/monkeys.py index 41cd3706..c8380711 100644 --- a/requests/monkeys.py +++ b/requests/monkeys.py @@ -9,7 +9,7 @@ Urllib2 Monkey patches. """ import urllib2 - +import re class Request(urllib2.Request): """Hidden wrapper around the urllib2.Request object. Allows for manual @@ -26,8 +26,9 @@ class Request(urllib2.Request): return urllib2.Request.get_method(self) -class HTTPRedirectHandler(urllib2.HTTPRedirectHandler): +class HTTPRedirectHandler(urllib2.HTTPRedirectHandler): + """HTTP Redirect handler.""" def http_error_301(self, req, fp, code, msg, headers): pass @@ -36,10 +37,13 @@ class HTTPRedirectHandler(urllib2.HTTPRedirectHandler): class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler): + """HTTP Basic Auth Handler with authentication loop fixes.""" def __init__(self, *args, **kwargs): urllib2.HTTPBasicAuthHandler.__init__(self, *args, **kwargs) self.retried_req = None + self.retried = 0 + def reset_retry_count(self): # Python 2.6.5 will call this on 401 or 407 errors and thus loop @@ -47,6 +51,7 @@ class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler): # http_error_auth_reqed instead. pass + def http_error_auth_reqed(self, auth_header, host, req, headers): # Reset the retry counter once for each request. if req is not self.retried_req: @@ -59,6 +64,59 @@ class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler): +class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler): + """HTTP Basic Auth Handler with forced Authentication.""" + + auth_header = 'Authorization' + rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+' + 'realm=(["\'])(.*?)\\2', re.I) + + def __init__(self, *args, **kwargs): + HTTPBasicAuthHandler.__init__(self, *args, **kwargs) + + + def http_error_401(self, req, fp, code, msg, headers): + url = req.get_full_url() + response = self._http_error_auth_reqed('www-authenticate', url, req, headers) + self.reset_retry_count() + return response + + http_error_404 = http_error_401 + + + def _http_error_auth_reqed(self, authreq, host, req, headers): + + authreq = headers.get(authreq, None) + + if self.retried > 5: + # retry sending the username:password 5 times before failing. + raise urllib2.HTTPError(req.get_full_url(), 401, "basic auth failed", + headers, None) + else: + self.retried += 1 + + if authreq: + + mo = self.rx.search(authreq) + + if mo: + scheme, quote, realm = mo.groups() + + if scheme.lower() == 'basic': + response = self.retry_http_basic_auth(host, req, realm) + + if response and response.code not in (401, 404): + self.retried = 0 + return response + else: + response = self.retry_http_basic_auth(host, req, 'Realm') + + if response and response.code not in (401, 404): + self.retried = 0 + return response + + + class HTTPDigestAuthHandler(urllib2.HTTPDigestAuthHandler): def __init__(self, *args, **kwargs): diff --git a/requests/structures.py b/requests/structures.py index bfee7b19..dd5168cf 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -9,20 +9,39 @@ Datastructures that power Requests. """ class CaseInsensitiveDict(dict): - """Case-insensitive Dictionary for :class:`Response ` Headers. + """Case-insensitive Dictionary For example, ``headers['content-encoding']`` will return the value of a ``'Content-Encoding'`` response header.""" - def _lower_keys(self): - return map(str.lower, self.keys()) + @property + def lower_keys(self): + if not hasattr(self, '_lower_keys') or not self._lower_keys: + self._lower_keys = dict((k.lower(), k) for k in self.iterkeys()) + return self._lower_keys + def _clear_lower_keys(self): + if hasattr(self, '_lower_keys'): + self._lower_keys.clear() + + def __setitem__(self, key, value): + dict.__setitem__(self, key, value) + self._clear_lower_keys() + + def __delitem__(self, key): + dict.__delitem__(self, key) + self._lower_keys.clear() def __contains__(self, key): - return key.lower() in self._lower_keys() - + return key.lower() in self.lower_keys def __getitem__(self, key): # We allow fall-through here, so values default to None if key in self: - return self.items()[self._lower_keys().index(key.lower())][1] + return dict.__getitem__(self, self.lower_keys[key.lower()]) + + def get(self, key, default=None): + if key in self: + return self[key] + else: + return default diff --git a/test_requests.py b/test_requests.py index f2aded24..8c137fad 100755 --- a/test_requests.py +++ b/test_requests.py @@ -6,7 +6,10 @@ from __future__ import with_statement import unittest import cookielib -import omnijson as json +try: + import omnijson as json +except ImportError: + import json import requests @@ -323,8 +326,41 @@ class RequestsTestSuite(unittest.TestCase): def test_idna(self): r = requests.get(u'http://➡.ws/httpbin') - self.assertEqual(r.url, HTTPBIN_URL) + assert 'httpbin' in r.url + def test_urlencoded_get_query_multivalued_param(self): + r = requests.get(httpbin('get'), params=dict(test=['foo','baz'])) + self.assertEquals(r.status_code, 200) + self.assertEquals(r.url, httpbin('get?test=foo&test=baz')) + + def test_urlencoded_post_querystring_multivalued(self): + r = requests.post(httpbin('post'), params=dict(test=['foo','baz'])) + self.assertEquals(r.status_code, 200) + self.assertEquals(r.headers['content-type'], 'application/json') + self.assertEquals(r.url, httpbin('post?test=foo&test=baz')) + rbody = json.loads(r.content) + self.assertEquals(rbody.get('form'), {}) # No form supplied + self.assertEquals(rbody.get('data'), '') + + def test_urlencoded_post_query_multivalued_and_data(self): + r = requests.post(httpbin('post'), params=dict(test=['foo','baz']), + data=dict(test2="foobar",test3=['foo','baz'])) + self.assertEquals(r.status_code, 200) + self.assertEquals(r.headers['content-type'], 'application/json') + self.assertEquals(r.url, httpbin('post?test=foo&test=baz')) + rbody = json.loads(r.content) + self.assertEquals(rbody.get('form'), dict(test2='foobar',test3='foo')) + self.assertEquals(rbody.get('data'), '') + + + def test_redirect_history(self): + r = requests.get(httpbin('redirect', '3')) + self.assertEquals(r.status_code, 200) + self.assertEquals(len(r.history), 3) + + r = requests.get(httpsbin('redirect', '3')) + self.assertEquals(r.status_code, 200) + self.assertEquals(len(r.history), 3) if __name__ == '__main__': unittest.main()