From b8d4a5ea61de8b579aa3e3d8e37e43f3cc34b619 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 10:04:46 -0400 Subject: [PATCH 01/24] v0.7.1 plans --- HISTORY.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 0db8550f..44da9090 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,13 @@ History ------- +0.7.1 ++++++ + +* Move away from urllib2 authentication. +* Remove AuthManager. + + 0.7.0 (2011-10-22) ++++++++++++++++++ From 340f7a0733b6b023c2bec18bb0f91e8f1052a84d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 10:56:04 -0400 Subject: [PATCH 02/24] initial auth module w/ httpbasic auth --- requests/auth.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 requests/auth.py diff --git a/requests/auth.py b/requests/auth.py new file mode 100644 index 00000000..900a048f --- /dev/null +++ b/requests/auth.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +""" +requests.auth +~~~~~~~~~~~~~ + +This module contains the authentication handlers for Requests. +""" + +from base64 import encodestring as base64 + +def http_basic(r, username, password): + """Attaches HTTP Basic Authentication to the given Request object. + Arguments should be considered non-positional. + + """ + + auth_s = base64('%s:%s' % (username, password)).replace('\n', '') + r.headers['Authorization'] = ('Basic %s' % auth_s) + + return r + + +def http_digest(r, username, password): + """Attaches HTTP Digest Authentication to the given Request object. + Arguments should be considered non-positional. + """ + + r.headers \ No newline at end of file From 94c587d642eb131660e738667dc3861fbd35314e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 11:00:21 -0400 Subject: [PATCH 03/24] Comment out authentication handlers --- requests/models.py | 175 ++++----------------------------------------- 1 file changed, 15 insertions(+), 160 deletions(-) diff --git a/requests/models.py b/requests/models.py index a6a2c01e..852dbbda 100644 --- a/requests/models.py +++ b/requests/models.py @@ -94,10 +94,10 @@ class Request(object): #: content and metadata of HTTP Response, once :attr:`sent `. self.response = Response() - if isinstance(auth, (list, tuple)): - auth = AuthObject(*auth) - if not auth: - auth = auth_manager.get_auth(self.url) + # if isinstance(auth, (list, tuple)): + # auth = AuthObject(*auth) + # if not auth: + # auth = auth_manager.get_auth(self.url) #: :class:`AuthObject` to attach to :class:`Request `. self.auth = auth @@ -138,21 +138,21 @@ class Request(object): if self.cookies is not None: _handlers.append(urllib2.HTTPCookieProcessor(self.cookies)) - if self.auth: - if not isinstance(self.auth.handler, - (urllib2.AbstractBasicAuthHandler, - urllib2.AbstractDigestAuthHandler)): + # if self.auth: + # if not isinstance(self.auth.handler, + # (urllib2.AbstractBasicAuthHandler, + # urllib2.AbstractDigestAuthHandler)): # TODO: REMOVE THIS COMPLETELY - auth_manager.add_password( - self.auth.realm, self.url, - self.auth.username, - self.auth.password) + # auth_manager.add_password( + # self.auth.realm, self.url, + # self.auth.username, + # self.auth.password) - self.auth.handler = self.auth.handler(auth_manager) - auth_manager.add_auth(self.url, self.auth) + # self.auth.handler = self.auth.handler(auth_manager) + # auth_manager.add_auth(self.url, self.auth) - _handlers.append(self.auth.handler) + # _handlers.append(self.auth.handler) if self.proxies: _handlers.append(urllib2.ProxyHandler(self.proxies)) @@ -522,151 +522,6 @@ class Response(object): -class AuthManager(object): - """Requests Authentication Manager.""" - - def __new__(cls): - singleton = cls.__dict__.get('__singleton__') - if singleton is not None: - return singleton - - cls.__singleton__ = singleton = object.__new__(cls) - - return singleton - - - def __init__(self): - self.passwd = {} - self._auth = {} - - - def __repr__(self): - return '' % (self.method) - - - def add_auth(self, uri, auth): - """Registers AuthObject to AuthManager.""" - - uri = self.reduce_uri(uri, False) - - # try to make it an AuthObject - if not isinstance(auth, AuthObject): - try: - auth = AuthObject(*auth) - except TypeError: - pass - - self._auth[uri] = auth - - - def add_password(self, realm, uri, user, passwd): - """Adds password to AuthManager.""" - # uri could be a single URI or a sequence - if isinstance(uri, basestring): - uri = [uri] - - reduced_uri = tuple([self.reduce_uri(u, False) for u in uri]) - - if reduced_uri not in self.passwd: - self.passwd[reduced_uri] = {} - self.passwd[reduced_uri] = (user, passwd) - - - def find_user_password(self, realm, authuri): - for uris, authinfo in self.passwd.iteritems(): - reduced_authuri = self.reduce_uri(authuri, False) - for uri in uris: - if self.is_suburi(uri, reduced_authuri): - return authinfo - - return (None, None) - - - def get_auth(self, uri): - (in_domain, in_path) = self.reduce_uri(uri, False) - - for domain, path, authority in ( - (i[0][0], i[0][1], i[1]) for i in self._auth.iteritems() - ): - if in_domain == domain: - if path in in_path: - return authority - - - def reduce_uri(self, uri, default_port=True): - """Accept authority or URI and extract only the authority and path.""" - - # note HTTP URLs do not have a userinfo component - parts = urllib2.urlparse.urlsplit(uri) - - if parts[1]: - # URI - scheme = parts[0] - authority = parts[1] - path = parts[2] or '/' - else: - # host or host:port - scheme = None - authority = uri - path = '/' - - host, port = urllib2.splitport(authority) - - if default_port and port is None and scheme is not None: - dport = {"http": 80, - "https": 443, - }.get(scheme) - if dport is not None: - authority = "%s:%d" % (host, dport) - - return authority, path - - - def is_suburi(self, base, test): - """Check if test is below base in a URI tree - - Both args must be URIs in reduced form. - """ - if base == test: - return True - if base[0] != test[0]: - return False - common = urllib2.posixpath.commonprefix((base[1], test[1])) - if len(common) == len(base[1]): - return True - return False - - - def empty(self): - self.passwd = {} - - - def remove(self, uri, realm=None): - # uri could be a single URI or a sequence - if isinstance(uri, basestring): - uri = [uri] - - for default_port in True, False: - reduced_uri = tuple([self.reduce_uri(u, default_port) for u in uri]) - del self.passwd[reduced_uri][realm] - - - def __contains__(self, uri): - # uri could be a single URI or a sequence - if isinstance(uri, basestring): - uri = [uri] - - uri = tuple([self.reduce_uri(u, False) for u in uri]) - - if uri in self.passwd: - return True - - return False - -auth_manager = AuthManager() - - - class AuthObject(object): """The :class:`AuthObject` is a simple HTTP Authentication token. When given to a Requests function, it enables Basic HTTP Authentication for that From d23c4b94c11eadfba7554fa7216f2a30b04396ca Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 11:00:36 -0400 Subject: [PATCH 04/24] httpbasic authentication cleanup --- requests/auth.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/requests/auth.py b/requests/auth.py index 900a048f..0a0402b2 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -7,15 +7,17 @@ requests.auth This module contains the authentication handlers for Requests. """ -from base64 import encodestring as base64 +from base64 import base64 def http_basic(r, username, password): """Attaches HTTP Basic Authentication to the given Request object. Arguments should be considered non-positional. """ + username = str(username) + password = str(password) - auth_s = base64('%s:%s' % (username, password)).replace('\n', '') + auth_s = base64('%s:%s' % (username, password)) r.headers['Authorization'] = ('Basic %s' % auth_s) return r From 0dae52220b274890af08ac040bd90747146c3d90 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 11:14:59 -0400 Subject: [PATCH 05/24] fix base64 encoding for httpbasic --- requests/auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/auth.py b/requests/auth.py index 0a0402b2..3a978b13 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -7,7 +7,7 @@ requests.auth This module contains the authentication handlers for Requests. """ -from base64 import base64 +from base64 import b64encode def http_basic(r, username, password): """Attaches HTTP Basic Authentication to the given Request object. @@ -17,7 +17,7 @@ def http_basic(r, username, password): username = str(username) password = str(password) - auth_s = base64('%s:%s' % (username, password)) + auth_s = b64encode('%s:%s' % (username, password)) r.headers['Authorization'] = ('Basic %s' % auth_s) return r From 06a5aed8e2a539d780020fb893fdf5ef9192ae04 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 11:15:11 -0400 Subject: [PATCH 06/24] assert sessions are using new auth --- test_requests.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test_requests.py b/test_requests.py index f1211571..984dc05f 100755 --- a/test_requests.py +++ b/test_requests.py @@ -147,11 +147,14 @@ class RequestsTestSuite(unittest.TestCase): url = service('basic-auth', 'user', 'pass') r = requests.get(url, auth=auth) - # print r.__dict__ self.assertEqual(r.status_code, 200) - r = requests.get(url) + self.assertEqual(r.status_code, 401) + + + s = requests.session(auth=auth) + r = s.get(url) self.assertEqual(r.status_code, 200) From 495c3d500640ce39d53292fd9846a2354d203675 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 11:15:42 -0400 Subject: [PATCH 07/24] Use new authentication style in models :metal: --- requests/models.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index 852dbbda..cb10175c 100644 --- a/requests/models.py +++ b/requests/models.py @@ -94,7 +94,9 @@ class Request(object): #: content and metadata of HTTP Response, once :attr:`sent `. self.response = Response() - # if isinstance(auth, (list, tuple)): + if isinstance(auth, (list, tuple)): + from .auth import http_basic + auth = (http_basic, auth) # auth = AuthObject(*auth) # if not auth: # auth = auth_manager.get_auth(self.url) @@ -138,7 +140,7 @@ class Request(object): if self.cookies is not None: _handlers.append(urllib2.HTTPCookieProcessor(self.cookies)) - # if self.auth: + # if not isinstance(self.auth.handler, # (urllib2.AbstractBasicAuthHandler, # urllib2.AbstractDigestAuthHandler)): @@ -348,6 +350,13 @@ class Request(object): data = self._enc_data headers = {} + if self.auth: + auth_func, auth_args = self.auth + + r = auth_func(self, *auth_args) + + self.__dict__.update(r.__dict__) + # Build the Urllib2 Request. req = _Request(url, data=data, headers=headers, method=self.method) From 4892a305d9cd658287b3320d18c60886b14b5b48 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 11:33:03 -0400 Subject: [PATCH 08/24] more advanced authentication mechanism --- requests/auth.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/requests/auth.py b/requests/auth.py index 3a978b13..6825441c 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -28,4 +28,32 @@ def http_digest(r, username, password): Arguments should be considered non-positional. """ - r.headers \ No newline at end of file + r.headers + + +def dispatch(t): + """Given an auth tuple, return an expanded version.""" + + if not t: + return t + else: + t = list(t) + + # Make sure they're passing in something. + assert len(t) <= 2 + + # If only two items are passed in, assume HTTPBasic. + if (len(t) == 2): + t.insert(0, 'basic') + + # Allow built-in string referenced auths. + if isinstance(t[0], basestring): + if t[0] in ('basic', 'forced_basic'): + t[0] = http_basic + elif t[0] in ('digest',): + t[0] = http_digest + + # Return a custom callable. + return (t[0], t[1:]) + + From 20e0ee4968298815ed8a3cd7de01a17e8451cd6c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 11:35:01 -0400 Subject: [PATCH 09/24] Remove AuthObject entirely. --- requests/models.py | 59 +++------------------------------------------- 1 file changed, 3 insertions(+), 56 deletions(-) diff --git a/requests/models.py b/requests/models.py index cb10175c..523245ba 100644 --- a/requests/models.py +++ b/requests/models.py @@ -27,6 +27,7 @@ from .monkeys import ( HTTPBasicAuthHandler, HTTPForcedBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler) +from .auth import dispatch as auth_dispatch REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) @@ -94,15 +95,8 @@ class Request(object): #: content and metadata of HTTP Response, once :attr:`sent `. self.response = Response() - if isinstance(auth, (list, tuple)): - from .auth import http_basic - auth = (http_basic, auth) - # auth = AuthObject(*auth) - # if not auth: - # auth = auth_manager.get_auth(self.url) - - #: :class:`AuthObject` to attach to :class:`Request `. - self.auth = auth + #: Authentication tuple to attach to :class:`Request `. + self.auth = auth_dispatch(auth) #: CookieJar to attach to :class:`Request `. self.cookies = cookies @@ -140,22 +134,6 @@ class Request(object): if self.cookies is not None: _handlers.append(urllib2.HTTPCookieProcessor(self.cookies)) - - # if not isinstance(self.auth.handler, - # (urllib2.AbstractBasicAuthHandler, - # urllib2.AbstractDigestAuthHandler)): - - # TODO: REMOVE THIS COMPLETELY - # auth_manager.add_password( - # self.auth.realm, self.url, - # self.auth.username, - # self.auth.password) - - # self.auth.handler = self.auth.handler(auth_manager) - # auth_manager.add_auth(self.url, self.auth) - - # _handlers.append(self.auth.handler) - if self.proxies: _handlers.append(urllib2.ProxyHandler(self.proxies)) @@ -529,34 +507,3 @@ class Response(object): if self.error: raise self.error - - -class AuthObject(object): - """The :class:`AuthObject` is a simple HTTP Authentication token. When - given to a Requests function, it enables Basic HTTP Authentication for that - Request. You can also enable Authorization for domain realms with AutoAuth. - See AutoAuth for more details. - - :param username: Username to authenticate with. - :param password: Password for given username. - :param realm: (optional) the realm this auth applies to - :param handler: (optional) basic || digest || proxy_basic || proxy_digest - """ - - _handlers = { - 'basic': HTTPBasicAuthHandler, - 'forced_basic': HTTPForcedBasicAuthHandler, - 'digest': HTTPDigestAuthHandler, - 'proxy_basic': urllib2.ProxyBasicAuthHandler, - 'proxy_digest': urllib2.ProxyDigestAuthHandler - } - - def __init__(self, username, password, handler='forced_basic', realm=None): - self.username = username - self.password = password - self.realm = realm - - if isinstance(handler, basestring): - self.handler = self._handlers.get(handler.lower(), HTTPForcedBasicAuthHandler) - else: - self.handler = handler From af6aa3e5729aa4e99716546915957ecf70bb894d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 12:26:41 -0400 Subject: [PATCH 10/24] Move hooks out of top-level loop --- requests/async.py | 6 ------ requests/sessions.py | 12 ------------ 2 files changed, 18 deletions(-) diff --git a/requests/async.py b/requests/async.py index db25f6a5..e14ac946 100644 --- a/requests/async.py +++ b/requests/async.py @@ -45,12 +45,6 @@ def _send(r, pools=None): r.send() - # Post-request hook. - r = dispatch_hook('post_request', r.hooks, r) - - # Response manipulation hook. - r.response = dispatch_hook('response', r.hooks, r.response) - return r.response diff --git a/requests/sessions.py b/requests/sessions.py index a88a0626..f1124ae2 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -157,14 +157,8 @@ class Session(object): args[attr] = merge_kwargs(local_val, session_val) - # Arguments manipulation hook. - args = dispatch_hook('args', hooks, args) - r = Request(**args) - # Pre-request hook. - r = dispatch_hook('pre_request', hooks, r) - # Don't send if asked nicely. if not return_response: return r @@ -172,12 +166,6 @@ class Session(object): # Send the HTTP Request. r.send() - # Post-request hook. - r = dispatch_hook('post_request', hooks, r) - - # Response manipulation hook. - r.response = dispatch_hook('response', hooks, r.response) - return r.response From 12db1576e9d0ed897f799a8f382ee92db891988d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 12:26:46 -0400 Subject: [PATCH 11/24] auth tuple fix --- requests/auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/auth.py b/requests/auth.py index 6825441c..d59a6874 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -40,7 +40,7 @@ def dispatch(t): t = list(t) # Make sure they're passing in something. - assert len(t) <= 2 + assert len(t) >= 2 # If only two items are passed in, assume HTTPBasic. if (len(t) == 2): @@ -54,6 +54,6 @@ def dispatch(t): t[0] = http_digest # Return a custom callable. - return (t[0], t[1:]) + return (t[0], tuple(t[1:])) From b7c788cc56b6f2f1113555074fad848afa9caf0f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 14:44:00 -0400 Subject: [PATCH 12/24] prettier async patching --- requests/async.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/requests/async.py b/requests/async.py index e14ac946..3b07f26e 100644 --- a/requests/async.py +++ b/requests/async.py @@ -28,7 +28,7 @@ __all__ = ( ) -def _patched(f): +def patched(f): """Patches a given API function to not send.""" def wrapped(*args, **kwargs): @@ -37,7 +37,7 @@ def _patched(f): return wrapped -def _send(r, pools=None): +def send(r, pools=None): """Sends a given Request object.""" if pools: @@ -49,13 +49,13 @@ def _send(r, pools=None): # Patched requests.api functions. -get = _patched(api.get) -head = _patched(api.head) -post = _patched(api.post) -put = _patched(api.put) -patch = _patched(api.patch) -delete = _patched(api.delete) -request = _patched(api.request) +get = patched(api.get) +head = patched(api.head) +post = patched(api.post) +put = patched(api.put) +patch = patched(api.patch) +delete = patched(api.delete) +request = patched(api.request) def map(requests, prefetch=True): @@ -65,7 +65,7 @@ def map(requests, prefetch=True): :param prefetch: If False, the content will not be downloaded immediately. """ - jobs = [gevent.spawn(_send, r) for r in requests] + jobs = [gevent.spawn(send, r) for r in requests] gevent.joinall(jobs) if prefetch: From 60b37e54b56ae9b1f3457bfda6178c001cdddcc4 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 14:44:20 -0400 Subject: [PATCH 13/24] Digest authentication support! Ripped from urllib2 --- requests/auth.py | 94 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/requests/auth.py b/requests/auth.py index d59a6874..e9dbce8b 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -7,7 +7,14 @@ requests.auth This module contains the authentication handlers for Requests. """ +import time +import hashlib + from base64 import b64encode +from urlparse import urlparse + +from .utils import randombytes, parse_dict_header + def http_basic(r, username, password): """Attaches HTTP Basic Authentication to the given Request object. @@ -28,7 +35,92 @@ def http_digest(r, username, password): Arguments should be considered non-positional. """ - r.headers + def handle_401(r): + """Takes the given response and tries digest-auth, if needed.""" + + s_auth = r.headers.get('www-authenticate', '') + + if 'digest' in s_auth.lower(): + + last_nonce = '' + nonce_count = 0 + + chal = parse_dict_header(s_auth.replace('Digest ', '')) + + realm = chal['realm'] + nonce = chal['nonce'] + qop = chal.get('qop') + algorithm = chal.get('algorithm', 'MD5') + opaque = chal.get('opaque', None) + + algorithm = algorithm.upper() + # lambdas assume digest modules are imported at the top level + if algorithm == 'MD5': + H = lambda x: hashlib.md5(x).hexdigest() + elif algorithm == 'SHA': + H = lambda x: hashlib.sha1(x).hexdigest() + # XXX MD5-sess + KD = lambda s, d: H("%s:%s" % (s, d)) + + if H is None: + return None + + # XXX not implemented yet + entdig = None + path = urlparse(r.request.url).path + + A1 = "%s:%s:%s" % (username, realm, password) + A2 = "%s:%s" % (r.request.method, path) + + if qop == 'auth': + if nonce == last_nonce: + nonce_count += 1 + else: + nonce_count = 1 + last_nonce = nonce + + ncvalue = '%08x' % nonce_count + cnonce = (hashlib.sha1("%s:%s:%s:%s" % ( + nonce_count, nonce, time.ctime(), randombytes(8))) + .hexdigest()[:16] + ) + noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2)) + respdig = KD(H(A1), noncebit) + elif qop is None: + respdig = KD(H(A1), "%s:%s" % (nonce, H(A2))) + else: + # XXX handle auth-int. + return None + + # XXX should the partial digests be encoded too? + base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \ + 'response="%s"' % (username, realm, nonce, path, respdig) + if opaque: + base += ', opaque="%s"' % opaque + if entdig: + base += ', digest="%s"' % entdig + base += ', algorithm="%s"' % algorithm + if qop: + base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce) + + + r.request.headers['Authorization'] = 'Digest %s' % (base) + r.request.send(anyway=True) + _r = r.request.response + _r.history.append(r) + print _r.status_code + + # r.request.response + + print locals() + + print _r.headers + return _r + + return r + + r.hooks['response'] = handle_401 + return r def dispatch(t): From 4916b1c2cee365bb4975645defe3ed115b161b30 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 14:44:26 -0400 Subject: [PATCH 14/24] pep8 --- requests/defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/defaults.py b/requests/defaults.py index a5699765..951c056d 100644 --- a/requests/defaults.py +++ b/requests/defaults.py @@ -25,7 +25,7 @@ defaults = dict() defaults['base_headers'] = { 'User-Agent': 'python-requests/%s' % __version__, - 'Accept-Encoding': ', '.join([ 'identity', 'deflate', 'compress', 'gzip' ]), + 'Accept-Encoding': ', '.join(('identity', 'deflate', 'compress', 'gzip')), } defaults['proxies'] = {} From 491a3c075d341a048c50b4a989b87da0c2a75992 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 14:46:35 -0400 Subject: [PATCH 15/24] Move hooks into internal event loop --- requests/models.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/requests/models.py b/requests/models.py index 523245ba..bafe2550 100644 --- a/requests/models.py +++ b/requests/models.py @@ -16,6 +16,7 @@ from urllib2 import HTTPError from urlparse import urlparse, urlunparse, urljoin from datetime import datetime +from .hooks import dispatch_hook from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers @@ -121,6 +122,10 @@ class Request(object): self.headers = headers + # Pre-request hook. + r = dispatch_hook('pre_request', hooks, self) + self.__dict__.update(r.__dict__) + def __repr__(self): return '' % (self.method) @@ -386,6 +391,13 @@ class Request(object): self.sent = self.response.ok + # Response manipulation hook. + self.response = dispatch_hook('response', self.hooks, self.response) + + # Post-request hook. + r = dispatch_hook('post_request', self.hooks, self) + self.__dict__.update(r.__dict__) + return self.sent From 53c7b777355a42eaef1df0c88916d413d3e996ec Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 14:46:44 -0400 Subject: [PATCH 16/24] Digest Authentication test. --- test_requests.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/test_requests.py b/test_requests.py index 984dc05f..82f95954 100755 --- a/test_requests.py +++ b/test_requests.py @@ -139,7 +139,7 @@ class RequestsTestSuite(unittest.TestCase): self.assertEqual(r.status_code, 200) - def test_AUTH_HTTP_200_OK_GET(self): + def test_BASICAUTH_HTTP_200_OK_GET(self): for service in SERVICES: @@ -158,6 +158,24 @@ class RequestsTestSuite(unittest.TestCase): self.assertEqual(r.status_code, 200) + def test_DIGESTAUTH_HTTP_200_OK_GET(self): + + for service in SERVICES: + + auth = ('digest', 'user', 'pass') + url = service('digest-auth', 'auth', 'user', 'pass') + + r = requests.get(url, auth=auth) + self.assertEqual(r.status_code, 200) + + r = requests.get(url) + self.assertEqual(r.status_code, 401) + + + s = requests.session(auth=auth) + r = s.get(url) + self.assertEqual(r.status_code, 200) + def test_POSTBIN_GET_POST_FILES(self): for service in SERVICES: From 9966017a4976339c76b834eab8a10b4dfba474f1 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 14:46:58 -0400 Subject: [PATCH 17/24] Add new utilities from werkzeug --- requests/utils.py | 109 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/requests/utils.py b/requests/utils.py index 8933c364..bd7dead4 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -12,9 +12,103 @@ that are also useful for external consumption. import cgi import codecs import cookielib +import os +import random import re import zlib +from urllib2 import parse_http_list as _parse_list_header + + +# From mitsuhiko/werkzeug (used with permission). +def parse_list_header(value): + """Parse lists as described by RFC 2068 Section 2. + + In particular, parse comma-separated lists where the elements of + the list may include quoted-strings. A quoted-string could + contain a comma. A non-quoted string could have quotes in the + middle. Quotes are removed automatically after parsing. + + It basically works like :func:`parse_set_header` just that items + may appear multiple times and case sensitivity is preserved. + + The return value is a standard :class:`list`: + + >>> parse_list_header('token, "quoted value"') + ['token', 'quoted value'] + + To create a header from the :class:`list` again, use the + :func:`dump_header` function. + + :param value: a string with a list header. + :return: :class:`list` + """ + result = [] + for item in _parse_list_header(value): + if item[:1] == item[-1:] == '"': + item = unquote_header_value(item[1:-1]) + result.append(item) + return result + + +# From mitsuhiko/werkzeug (used with permission). +def parse_dict_header(value): + """Parse lists of key, value pairs as described by RFC 2068 Section 2 and + convert them into a python dict: + + >>> d = parse_dict_header('foo="is a fish", bar="as well"') + >>> type(d) is dict + True + >>> sorted(d.items()) + [('bar', 'as well'), ('foo', 'is a fish')] + + If there is no value for a key it will be `None`: + + >>> parse_dict_header('key_without_value') + {'key_without_value': None} + + To create a header from the :class:`dict` again, use the + :func:`dump_header` function. + + :param value: a string with a dict header. + :return: :class:`dict` + """ + result = {} + for item in _parse_list_header(value): + if '=' not in item: + result[item] = None + continue + name, value = item.split('=', 1) + if value[:1] == value[-1:] == '"': + value = unquote_header_value(value[1:-1]) + result[name] = value + return result + + +# From mitsuhiko/werkzeug (used with permission). +def unquote_header_value(value, is_filename=False): + r"""Unquotes a header value. (Reversal of :func:`quote_header_value`). + This does not use the real unquoting but what browsers are actually + using for quoting. + + :param value: the header value to unquote. + """ + if value and value[0] == value[-1] == '"': + # this is not the real unquoting, but fixing this so that the + # RFC is met will result in bugs with internet explorer and + # probably some other browsers as well. IE for example is + # uploading files with "C:\foo\bar.txt" as filename + value = value[1:-1] + + # if this is a filename and the starting characters look like + # a UNC path, then just return the value without quotes. Using the + # replace sequence below on a UNC path has the effect of turning + # the leading double slash into a single slash and then + # _fix_ie_filename() doesn't work correctly. See #458. + if not is_filename or value[:2] != '\\\\': + return value.replace('\\\\', '\\').replace('\\"', '"') + return value + def header_expand(headers): """Returns an HTTP Header value string from a dictionary. @@ -63,6 +157,21 @@ def header_expand(headers): +def randombytes(n): + """Return n random bytes.""" + # Use /dev/urandom if it is available. Fall back to random module + # if not. It might be worthwhile to extend this function to use + # other platform-specific mechanisms for getting random bytes. + if os.path.exists("/dev/urandom"): + f = open("/dev/urandom") + s = f.read(n) + f.close() + return s + else: + L = [chr(random.randrange(0, 256)) for i in range(n)] + return "".join(L) + + def dict_from_cookiejar(cj): """Returns a key/value dictionary from a CookieJar. From 68b48309480db3a95372967fe9c005d612c91ea0 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 14:47:15 -0400 Subject: [PATCH 18/24] Add argument injection hook back --- requests/sessions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requests/sessions.py b/requests/sessions.py index f1124ae2..9c7f32c7 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -157,6 +157,9 @@ class Session(object): args[attr] = merge_kwargs(local_val, session_val) + # Arguments manipulation hook. + args = dispatch_hook('args', hooks, args) + r = Request(**args) # Don't send if asked nicely. @@ -166,6 +169,7 @@ class Session(object): # Send the HTTP Request. r.send() + return r.response From b88fbb5e0c1a95987612e891c60c3b14418fd077 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 14:49:59 -0400 Subject: [PATCH 19/24] KILL THE AUTH HANDLERS --- requests/models.py | 4 +- requests/monkeys.py | 119 ++------------------------------------------ 2 files changed, 6 insertions(+), 117 deletions(-) diff --git a/requests/models.py b/requests/models.py index bafe2550..1fac1a8d 100644 --- a/requests/models.py +++ b/requests/models.py @@ -24,9 +24,7 @@ from .utils import (dict_from_cookiejar, get_unicode_from_response, stream_decod from .status_codes import codes from .exceptions import Timeout, URLRequired, TooManyRedirects from .monkeys import Request as _Request -from .monkeys import ( - HTTPBasicAuthHandler, HTTPForcedBasicAuthHandler, - HTTPDigestAuthHandler, HTTPRedirectHandler) +from .monkeys import HTTPRedirectHandler from .auth import dispatch as auth_dispatch diff --git a/requests/monkeys.py b/requests/monkeys.py index c8380711..5cea7cf2 100644 --- a/requests/monkeys.py +++ b/requests/monkeys.py @@ -29,120 +29,11 @@ class Request(urllib2.Request): class HTTPRedirectHandler(urllib2.HTTPRedirectHandler): """HTTP Redirect handler.""" - def http_error_301(self, req, fp, code, msg, headers): + def _pass(self, req, fp, code, msg, headers): pass - http_error_302 = http_error_303 = http_error_307 = http_error_301 + http_error_302 = _pass + http_error_303 = _pass + http_error_307 = _pass + http_error_301 = _pass - - -class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler): - """HTTP Basic Auth Handler with authentication loop fixes.""" - - def __init__(self, *args, **kwargs): - urllib2.HTTPBasicAuthHandler.__init__(self, *args, **kwargs) - self.retried_req = None - self.retried = 0 - - - def reset_retry_count(self): - # Python 2.6.5 will call this on 401 or 407 errors and thus loop - # forever. We disable reset_retry_count completely and reset in - # http_error_auth_reqed instead. - pass - - - def http_error_auth_reqed(self, auth_header, host, req, headers): - # Reset the retry counter once for each request. - if req is not self.retried_req: - self.retried_req = req - self.retried = 0 - - return urllib2.HTTPBasicAuthHandler.http_error_auth_reqed( - self, auth_header, host, req, headers - ) - - - -class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler): - """HTTP Basic Auth Handler with forced Authentication.""" - - auth_header = 'Authorization' - rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+' - 'realm=(["\'])(.*?)\\2', re.I) - - def __init__(self, *args, **kwargs): - HTTPBasicAuthHandler.__init__(self, *args, **kwargs) - - - def http_error_401(self, req, fp, code, msg, headers): - url = req.get_full_url() - response = self._http_error_auth_reqed('www-authenticate', url, req, headers) - self.reset_retry_count() - return response - - http_error_404 = http_error_401 - - - def _http_error_auth_reqed(self, authreq, host, req, headers): - - authreq = headers.get(authreq, None) - - if self.retried > 5: - # retry sending the username:password 5 times before failing. - raise urllib2.HTTPError(req.get_full_url(), 401, "basic auth failed", - headers, None) - else: - self.retried += 1 - - if authreq: - - mo = self.rx.search(authreq) - - if mo: - scheme, quote, realm = mo.groups() - - if scheme.lower() == 'basic': - response = self.retry_http_basic_auth(host, req, realm) - - if response and response.code not in (401, 404): - self.retried = 0 - return response - else: - response = self.retry_http_basic_auth(host, req, 'Realm') - - if response and response.code not in (401, 404): - self.retried = 0 - return response - - - -class HTTPDigestAuthHandler(urllib2.HTTPDigestAuthHandler): - - def __init__(self, *args, **kwargs): - urllib2.HTTPDigestAuthHandler.__init__(self, *args, **kwargs) - self.retried_req = None - - def reset_retry_count(self): - # Python 2.6.5 will call this on 401 or 407 errors and thus loop - # forever. We disable reset_retry_count completely and reset in - # http_error_auth_reqed instead. - pass - - def http_error_auth_reqed(self, auth_header, host, req, headers): - # Reset the retry counter once for each request. - if req is not self.retried_req: - self.retried_req = req - self.retried = 0 - # In python < 2.5 AbstractDigestAuthHandler raises a ValueError if - # it doesn't know about the auth type requested. This can happen if - # somebody is using BasicAuth and types a bad password. - - try: - return urllib2.HTTPDigestAuthHandler.http_error_auth_reqed( - self, auth_header, host, req, headers) - except ValueError, inst: - arg = inst.args[0] - if arg.startswith("AbstractDigestAuthHandler doesn't know "): - return - raise \ No newline at end of file From 82cbae479c248d565e86d6664a0cb89d39dece6b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 14:51:22 -0400 Subject: [PATCH 20/24] history for this --- HISTORY.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 44da9090..d6ddbd05 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -4,8 +4,9 @@ History 0.7.1 +++++ -* Move away from urllib2 authentication. -* Remove AuthManager. +* Move away from urllib2 authentication handling. +* Fully Remove AuthManager, AuthObject, &c. +* New tuple-based auth system with handler callbacks. 0.7.0 (2011-10-22) From e46251b727479f8c477424afaddc80d2eb388e67 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 14:52:20 -0400 Subject: [PATCH 21/24] ch ch changes --- HISTORY.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d6ddbd05..a6a02daa 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,8 +1,8 @@ History ------- -0.7.1 -+++++ +0.7.1 (2011-10-23) +++++++++++++++++++ * Move away from urllib2 authentication handling. * Fully Remove AuthManager, AuthObject, &c. From 8f8dbec0d5a48ab39350a37b2c558ed95a4c8252 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 14:59:46 -0400 Subject: [PATCH 22/24] docs for basic/digest auth --- docs/user/quickstart.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 7ec7593d..3adfb7ac 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -136,3 +136,32 @@ parameter:: >>> r = requests.get(url, cookies=cookies) >>> r.content '{"cookies": {"cookies_are": "working"}}' + + +Basic Authentication +-------------------- + +Most web services require authentication. There many different types of +authentication, but the most common is called HTTP Basic Auth. + +Making requests with Basic Auth is easy, with Requests:: + + >>> requests.get('https://api.github.com/user', auth=('user', 'pass')) + + + +Digest Authentication +--------------------- + +Another popular form of protecting web service is Digest Authentication. + +Requests supports it!:: + + >>> url = 'http://httpbin.org/digest-auth/auth/user/pass' + >>> requests.get(url, auth=('digest', 'user', 'pass')) + + + +----------------------- + +Ready for more? Check out the advanced_ section. \ No newline at end of file From 58b1c69b6a40749f7dcc03f6489e5486e50f7ea2 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 15:13:34 -0400 Subject: [PATCH 23/24] Explain custom auth handlers --- docs/user/advanced.rst | 33 +++++++++++++++++++++++++++++++++ requests/sessions.py | 2 +- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 5c17898e..90d11433 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -157,6 +157,39 @@ And give it a try:: } +Custom Authentication +--------------------- + +Requests allows you to use specify your own authentication mechanism. + +When you pass our authentication tuple to a request method, the first +string is the type of authentication. 'basic' is inferred if none is +provided. + +You can pass in a callable object instead of a string for the first item +in the tuple, and it will be used in place of the built in authentication +callbacks. + +Let's pretend that we have a web service that will only respond if the +``X-Pizza`` header is set to a password value. Unlikely, but just go with it. + +We simply need to define a callback function that will be used to update the +Request object, right before it is dispatched. + +:: + + def pizza_auth(r, username): + """Attaches HTTP Pizza Authentication to the given Request object. + """ + r.headers['X-Pizza'] = username + + return r + +Then, we can make a request using our Pizza Auth:: + + >>> requests.get('http://pizzabin.org/admin', auth=(pizza_auth, 'kenneth')) + + Verbose Logging --------------- diff --git a/requests/sessions.py b/requests/sessions.py index 9c7f32c7..c5cd5015 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -112,7 +112,7 @@ class Session(object): :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. - :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param auth: (optional) Auth typle to enable Basic/Digest/Custom HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. From 02cf3dce3786becb89f631185e8207080b0b726c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 15:18:27 -0400 Subject: [PATCH 24/24] v0.7.1 --- requests/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/__init__.py b/requests/__init__.py index fdaaf068..bf699c5a 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -15,8 +15,8 @@ requests """ __title__ = 'requests' -__version__ = '0.7.0' -__build__ = 0x000700 +__version__ = '0.7.1' +__build__ = 0x000701 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz'