From 430e87d0fd738adde494ccfe7d3fb3882fd8ca02 Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Mon, 15 Aug 2011 00:12:14 -0400 Subject: [PATCH 001/255] First commit of curl command from request --- requests/models.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/requests/models.py b/requests/models.py index 08f3e321..9e0b0ddb 100644 --- a/requests/models.py +++ b/requests/models.py @@ -328,6 +328,37 @@ class Request(object): return self.sent + @property + def curl(self): + """Creates a curl command from the request""" + + #TODO - Auth. User names and accounts + #TODO - Query string... + #TODO - Files...How do I do files??? + + #: --location - if there is a redirect, redo request on the new place + curl_cmd = 'curl --location ' + + if self.method.upper() == 'HEAD': + #: --head - fetch headers only + method_opt = '--head ' + else: + #: --request - specify request method + method_opt = '--request %s ' % self.method.upper() + + data = '' + if self.method in ('PUT', 'POST', 'PATCH'): + #: --data - send specified data in post request. + #: '-data name=John -data skill=Doe' generates the + #: post chunk 'name=daniel&skill=lousy' + + #if data is file: + if isinstance(self.data, (list, tuple)): + data = data.join(['--data ' + key + '=' + value + ' ' for key, value in self.data]) + + curl_cmd = curl_cmd + method_opt + data + self.url + + return curl_cmd class Response(object): From aad27218d58e82637c43f92c85237fa90f0178c6 Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Tue, 16 Aug 2011 01:39:02 -0400 Subject: [PATCH 002/255] Using build_url and also including headers --- requests/models.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index 9e0b0ddb..5e1474d5 100644 --- a/requests/models.py +++ b/requests/models.py @@ -333,12 +333,16 @@ class Request(object): """Creates a curl command from the request""" #TODO - Auth. User names and accounts - #TODO - Query string... #TODO - Files...How do I do files??? #: --location - if there is a redirect, redo request on the new place curl_cmd = 'curl --location ' + #: --header - Extra header to use when getting a web page + header = '' + if self.headers: + header = header.join(['--header "' + key + ':' + value + '" ' for key, value in self.headers.iteritems()]) + if self.method.upper() == 'HEAD': #: --head - fetch headers only method_opt = '--head ' @@ -355,8 +359,11 @@ class Request(object): #if data is file: if isinstance(self.data, (list, tuple)): data = data.join(['--data ' + key + '=' + value + ' ' for key, value in self.data]) + else: + data = '--data ' + self._enc_data + ' ' - curl_cmd = curl_cmd + method_opt + data + self.url + #: Params handled in _build_url + curl_cmd = curl_cmd + method_opt + data + header + '"' + self._build_url() + '"' return curl_cmd From 2dc556e0825bf3a235140c5af6b473945ebf4ab8 Mon Sep 17 00:00:00 2001 From: Luca De Vitis Date: Tue, 23 Aug 2011 15:06:22 +0200 Subject: [PATCH 003/255] Added multiple hooks support. --- requests/api.py | 10 +++++----- requests/hooks.py | 26 +++++++++++++++++--------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/requests/api.py b/requests/api.py index 0928d8c8..9acc1e5f 100644 --- a/requests/api.py +++ b/requests/api.py @@ -14,7 +14,7 @@ This module impliments the Requests API. import config from .models import Request, Response, AuthObject from .status_codes import codes -from .hooks import dispatch_hook +from .hooks import dispatch_hooks from .utils import cookiejar_from_dict from urlparse import urlparse @@ -63,21 +63,21 @@ def request(method, url, ) # Arguments manipulation hook. - args = dispatch_hook('args', hooks, args) + args = dispatch_hooks('args', hooks, args) r = Request(**args) # Pre-request hook. - r = dispatch_hook('pre_request', hooks, r) + r = dispatch_hooks('pre_request', hooks, r) # Send the HTTP Request. r.send() # Post-request hook. - r = dispatch_hook('post_request', hooks, r) + r = dispatch_hooks('post_request', hooks, r) # Response manipulation hook. - r.response = dispatch_hook('response', hooks, r.response) + r.response = dispatch_hooks('response', hooks, r.response) return r.response diff --git a/requests/hooks.py b/requests/hooks.py index 2938029b..2fa97d0b 100644 --- a/requests/hooks.py +++ b/requests/hooks.py @@ -23,18 +23,26 @@ Available hooks: """ import warnings +from collections import Iterable +def dispatch_hooks(key, hooks, hook_data): + """Dispatches multiple hooks on a given piece of data. -def dispatch_hook(key, hooks, hook_data): - """Dipatches a hook dictionary on a given peice of data.""" - - hooks = hooks or dict() - - if key in hooks: + :param key: the hooks group to lookup + :type key: str + :param hooks: the hooks dictionary. The value of each key can be a callable + object, or a list of callable objects. + :type hooks: dict + :param hook_data: the object on witch the hooks should be applied + :type hook_data: object + """ + hook_list = hooks.get(key, []) if hooks else [] + dispatching = hook_list if isinstance(hook_list, Iterable) else [hook_list] + for hook in dispatching: try: - return hooks.get(key).__call__(hook_data) or hook_data - + # hook must be a callable + hook_data = hook(hook_data) except Exception, why: warnings.warn(str(why)) - return hook_data + From d12eac8669bc820c7fdd4efe47032112a6b41a1b Mon Sep 17 00:00:00 2001 From: Luca De Vitis Date: Tue, 23 Aug 2011 23:30:50 +0200 Subject: [PATCH 004/255] * hooks is a package now. * Added hooks.setup_hooks to setup multiple and default hooks * Added hooks.response hooks to unicode encoding, and uncompress response. * Modified hooks.dispatch_hooks api: don't need a key any more. --- requests/api.py | 16 +++++++-------- requests/config.py | 4 +++- requests/{hooks.py => hooks/__init__.py} | 23 ++++++++++++++++----- requests/models.py | 26 +++++++++++------------- 4 files changed, 41 insertions(+), 28 deletions(-) rename requests/{hooks.py => hooks/__init__.py} (61%) diff --git a/requests/api.py b/requests/api.py index 9acc1e5f..9c78c17f 100644 --- a/requests/api.py +++ b/requests/api.py @@ -14,7 +14,7 @@ This module impliments the Requests API. import config from .models import Request, Response, AuthObject from .status_codes import codes -from .hooks import dispatch_hooks +from hooks import setup_hooks, dispatch_hooks from .utils import cookiejar_from_dict from urlparse import urlparse @@ -43,10 +43,9 @@ def request(method, url, method = str(method).upper() - if cookies is None: - cookies = {} + cookies = cookiejar_from_dict(cookies or dict()) - cookies = cookiejar_from_dict(cookies) + hooks = setup_hooks(hooks or dict()) args = dict( method = method, @@ -61,23 +60,24 @@ def request(method, url, allow_redirects = allow_redirects, proxies = proxies or config.settings.proxies, ) + # Arguments manipulation hook. - args = dispatch_hooks('args', hooks, args) + args = dispatch_hooks(hooks.get('args', []), args) r = Request(**args) # Pre-request hook. - r = dispatch_hooks('pre_request', hooks, r) + r = dispatch_hooks(hooks.get('pre_request', []), r) # Send the HTTP Request. r.send() # Post-request hook. - r = dispatch_hooks('post_request', hooks, r) + r = dispatch_hooks(hooks.get('post_request', []), hooks, r) # Response manipulation hook. - r.response = dispatch_hooks('response', hooks, r.response) + r.response = dispatch_hooks(hooks.get('response', []), r.response) return r.response diff --git a/requests/config.py b/requests/config.py index 794109c5..a92e1f57 100644 --- a/requests/config.py +++ b/requests/config.py @@ -62,7 +62,9 @@ settings.proxies = None settings.verbose = None settings.timeout = None settings.max_redirects = 30 -settings.decode_unicode = True +# settings.decode_unicode = True +settings.unicode_response = True +settings.decode_response = True #: Use socket.setdefaulttimeout() as fallback? settings.timeout_fallback = True diff --git a/requests/hooks.py b/requests/hooks/__init__.py similarity index 61% rename from requests/hooks.py rename to requests/hooks/__init__.py index 2fa97d0b..8f7c6b1c 100644 --- a/requests/hooks.py +++ b/requests/hooks/__init__.py @@ -24,8 +24,24 @@ Available hooks: import warnings from collections import Iterable +from .. import config +from response import unicode_response, decode_response -def dispatch_hooks(key, hooks, hook_data): +def setup_hooks(hooks): + """Setup hooks as a dictionary. Each value is a set of hooks.""" + + for key, values in hooks.items(): + hook_list = values if isinstance(values, Iterable) else [values] + hooks[key] = set(hook_list) + + # Also, based on settings, + if config.settings.unicode_response: + hooks.setdefault('response', set()).add(unicode_response) + if config.settings.decode_response: + hooks.setdefault('response', set()).add(decode_response) + return hooks + +def dispatch_hooks(hooks, hook_data): """Dispatches multiple hooks on a given piece of data. :param key: the hooks group to lookup @@ -36,13 +52,10 @@ def dispatch_hooks(key, hooks, hook_data): :param hook_data: the object on witch the hooks should be applied :type hook_data: object """ - hook_list = hooks.get(key, []) if hooks else [] - dispatching = hook_list if isinstance(hook_list, Iterable) else [hook_list] - for hook in dispatching: + for hook in hooks: try: # hook must be a callable hook_data = hook(hook_data) except Exception, why: warnings.warn(str(why)) return hook_data - diff --git a/requests/models.py b/requests/models.py index 5983c237..e199b3ca 100644 --- a/requests/models.py +++ b/requests/models.py @@ -444,22 +444,20 @@ class Response(object): (if available). """ - if self._content is not None: - return self._content + if self._content is None: + # Read the contents. + self._content = self.fo.read() - # Read the contents. - self._content = self.fo.read() + # # Decode GZip'd content. + # if 'gzip' in self.headers.get('content-encoding', ''): + # try: + # self._content = decode_gzip(self._content) + # except zlib.error: + # pass - # Decode GZip'd content. - if 'gzip' in self.headers.get('content-encoding', ''): - try: - self._content = decode_gzip(self._content) - except zlib.error: - pass - - # Decode unicode content. - if settings.decode_unicode: - self._content = get_unicode_from_response(self) + # # Decode unicode content. + # if settings.decode_unicode: + # self._content = get_unicode_from_response(self) return self._content From 660e0903157579fae39d72c1ac7af45a1f6dc9d1 Mon Sep 17 00:00:00 2001 From: Luca De Vitis Date: Tue, 23 Aug 2011 23:39:07 +0200 Subject: [PATCH 005/255] Forgot to add the hooks sub-package --- requests/hooks/args.py | 7 ++++++ requests/hooks/post_request.py | 7 ++++++ requests/hooks/pre_request.py | 7 ++++++ requests/hooks/response.py | 43 ++++++++++++++++++++++++++++++++++ 4 files changed, 64 insertions(+) create mode 100644 requests/hooks/args.py create mode 100644 requests/hooks/post_request.py create mode 100644 requests/hooks/pre_request.py create mode 100644 requests/hooks/response.py diff --git a/requests/hooks/args.py b/requests/hooks/args.py new file mode 100644 index 00000000..6ac6fb6c --- /dev/null +++ b/requests/hooks/args.py @@ -0,0 +1,7 @@ +""" +request.hooks.args +~~~~~~~~~~~~~~~~~~ + +This module provide a collection of args hooks. +""" + diff --git a/requests/hooks/post_request.py b/requests/hooks/post_request.py new file mode 100644 index 00000000..3a4ff543 --- /dev/null +++ b/requests/hooks/post_request.py @@ -0,0 +1,7 @@ +""" +request.hooks.post_request +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This module provide a collection of post_request hooks. +""" + diff --git a/requests/hooks/pre_request.py b/requests/hooks/pre_request.py new file mode 100644 index 00000000..ca1e715f --- /dev/null +++ b/requests/hooks/pre_request.py @@ -0,0 +1,7 @@ +""" +request.hooks.pre_request +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This module provide a collection of pre_request hooks. +""" + diff --git a/requests/hooks/response.py b/requests/hooks/response.py new file mode 100644 index 00000000..fc3d26dc --- /dev/null +++ b/requests/hooks/response.py @@ -0,0 +1,43 @@ +""" +request.hooks.response +~~~~~~~~~~~~~~~~~~~~~~ + +This module provide a collection of response hooks. +""" +from functools import wraps +import zlib +from cgi import parse_header + +#: Dictionary of content decoders. +decoders = { + # No decoding applied. + 'identity': lambda r: r, + # Decode Response file object compressed with deflate. + 'deflate': lambda r: zlib.decompress(r.content), + # Decode Response file object compressed with gzip. + 'gzip': lambda r: zlib.decompress(r.content, 16+zlib.MAX_WBITS), +} + +# Decode Response file object compressed with compress. +decoders['compress'] = decoders['deflate'] + +try: + import bz2 +except ImportError: + pass +else: + # Decode Response file object compressed with bz2. + decoders['bzip2'] = lambda r: bz2.decompress(r.content) + +def unicode_response(r): + """Encode response file object in unicode.""" + content_type, params = parse_header(r.headers.get('content-type')) + charset = params.get('charset', '').strip("'\"") + r.content = unicode(r.content, charset) if charset else unicode(r.content) + return r + +def decode_response(r): + """Decode compressed response content using Contetn-Encoding header.""" + encoding = r.headers.get('content-encoding') + return decoders.get(encoding)(r) + From 56406b3442b20a3ea5536cabbd10477cc36244b8 Mon Sep 17 00:00:00 2001 From: Luca De Vitis Date: Tue, 23 Aug 2011 23:43:40 +0200 Subject: [PATCH 006/255] hooks now comply hook interface --- requests/hooks/response.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/requests/hooks/response.py b/requests/hooks/response.py index fc3d26dc..e9ce39b3 100644 --- a/requests/hooks/response.py +++ b/requests/hooks/response.py @@ -11,7 +11,7 @@ from cgi import parse_header #: Dictionary of content decoders. decoders = { # No decoding applied. - 'identity': lambda r: r, + 'identity': lambda r: r.content, # Decode Response file object compressed with deflate. 'deflate': lambda r: zlib.decompress(r.content), # Decode Response file object compressed with gzip. @@ -33,11 +33,12 @@ def unicode_response(r): """Encode response file object in unicode.""" content_type, params = parse_header(r.headers.get('content-type')) charset = params.get('charset', '').strip("'\"") - r.content = unicode(r.content, charset) if charset else unicode(r.content) + r._content = unicode(r.content, charset) if charset else unicode(r.content) return r def decode_response(r): """Decode compressed response content using Contetn-Encoding header.""" encoding = r.headers.get('content-encoding') - return decoders.get(encoding)(r) + r._content = decoders.get(encoding)(r) + return r From 2a17e5237c90a537c96b486ef30cddf8b544a966 Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Tue, 23 Aug 2011 22:03:42 -0400 Subject: [PATCH 007/255] Ive never seen anyone use the --'s unix command options --- requests/models.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/requests/models.py b/requests/models.py index 5e1474d5..9c51bb8a 100644 --- a/requests/models.py +++ b/requests/models.py @@ -332,23 +332,24 @@ class Request(object): def curl(self): """Creates a curl command from the request""" - #TODO - Auth. User names and accounts - #TODO - Files...How do I do files??? + #TODO - Auth with User names and accounts + #TODO - Files + #TODO - OAuth - #: --location - if there is a redirect, redo request on the new place - curl_cmd = 'curl --location ' + #: -L/--location - if there is a redirect, redo request on the new place + curl_cmd = 'curl -L ' - #: --header - Extra header to use when getting a web page + #: -H/--header - Extra header to use when getting a web page header = '' if self.headers: - header = header.join(['--header "' + key + ':' + value + '" ' for key, value in self.headers.iteritems()]) + header = header.join(['-H "' + key + ':' + value + '" ' for key, value in self.headers.iteritems()]) if self.method.upper() == 'HEAD': - #: --head - fetch headers only - method_opt = '--head ' + #: -I/--head - fetch headers only + method_opt = '-I ' else: - #: --request - specify request method - method_opt = '--request %s ' % self.method.upper() + #: -X/--request - specify request method + method_opt = '-X %s ' % self.method.upper() data = '' if self.method in ('PUT', 'POST', 'PATCH'): @@ -358,9 +359,9 @@ class Request(object): #if data is file: if isinstance(self.data, (list, tuple)): - data = data.join(['--data ' + key + '=' + value + ' ' for key, value in self.data]) + data = data.join(['-d ' + key + '=' + value + ' ' for key, value in self.data]) else: - data = '--data ' + self._enc_data + ' ' + data = '-d ' + self._enc_data + ' ' #: Params handled in _build_url curl_cmd = curl_cmd + method_opt + data + header + '"' + self._build_url() + '"' From c56acf0e321d7a2ae1c0726352d9d7698091d05c Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Tue, 23 Aug 2011 22:17:36 -0400 Subject: [PATCH 008/255] Small Cleanup --- requests/models.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/requests/models.py b/requests/models.py index 9c51bb8a..ec18b08c 100644 --- a/requests/models.py +++ b/requests/models.py @@ -353,20 +353,17 @@ class Request(object): data = '' if self.method in ('PUT', 'POST', 'PATCH'): - #: --data - send specified data in post request. - #: '-data name=John -data skill=Doe' generates the - #: post chunk 'name=daniel&skill=lousy' + #: -d/--data - send specified data in post request. + #: '-d name=John -d last=Doe' generates the + #: post chunk 'name=John&last=Doe' - #if data is file: if isinstance(self.data, (list, tuple)): data = data.join(['-d ' + key + '=' + value + ' ' for key, value in self.data]) else: data = '-d ' + self._enc_data + ' ' #: Params handled in _build_url - curl_cmd = curl_cmd + method_opt + data + header + '"' + self._build_url() + '"' - - return curl_cmd + return curl_cmd + method_opt + data + header + '"' + self._build_url() + '"' class Response(object): From 84d9ff0c5c63ec22dd10670e373b4a46d8770ff4 Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Tue, 23 Aug 2011 22:34:05 -0400 Subject: [PATCH 009/255] More Small Cleanup --- requests/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/requests/models.py b/requests/models.py index 7a9ba098..3cd8def1 100644 --- a/requests/models.py +++ b/requests/models.py @@ -421,6 +421,7 @@ class Request(object): #: Params handled in _build_url return curl_cmd + method_opt + data + header + '"' + self._build_url() + '"' + class Response(object): """The core :class:`Response ` object. All :class:`Request ` objects contain a From 0bf60c8876cddb5b203c652834ab8c6da55e4ce5 Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Tue, 23 Aug 2011 23:08:26 -0400 Subject: [PATCH 010/255] Adding check when _enc_data in None --- requests/models.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/requests/models.py b/requests/models.py index 3cd8def1..fc828e13 100644 --- a/requests/models.py +++ b/requests/models.py @@ -391,6 +391,7 @@ class Request(object): #TODO - Auth with User names and accounts #TODO - Files #TODO - OAuth + #TODO - Cookies? #: -L/--location - if there is a redirect, redo request on the new place curl_cmd = 'curl -L ' @@ -410,12 +411,9 @@ class Request(object): data = '' if self.method in ('PUT', 'POST', 'PATCH'): #: -d/--data - send specified data in post request. - #: '-d name=John -d last=Doe' generates the - #: post chunk 'name=John&last=Doe' - if isinstance(self.data, (list, tuple)): data = data.join(['-d ' + key + '=' + value + ' ' for key, value in self.data]) - else: + elif self._enc_data is not None: data = '-d ' + self._enc_data + ' ' #: Params handled in _build_url From 256c86f8f7deb5375c38b6cc55c3d54d0366d283 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 23 Aug 2011 23:52:43 -0400 Subject: [PATCH 011/255] moving curl into utils #139 --- requests/models.py | 35 ----------------------------------- requests/utils.py | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/requests/models.py b/requests/models.py index 0d90ea92..5ff9eb48 100644 --- a/requests/models.py +++ b/requests/models.py @@ -383,41 +383,6 @@ class Request(object): return self.sent - @property - def curl(self): - """Creates a curl command from the request""" - - #TODO - Auth with User names and accounts - #TODO - Files - #TODO - OAuth - #TODO - Cookies? - - #: -L/--location - if there is a redirect, redo request on the new place - curl_cmd = 'curl -L ' - - #: -H/--header - Extra header to use when getting a web page - header = '' - if self.headers: - header = header.join(['-H "' + key + ':' + value + '" ' for key, value in self.headers.iteritems()]) - - if self.method.upper() == 'HEAD': - #: -I/--head - fetch headers only - method_opt = '-I ' - else: - #: -X/--request - specify request method - method_opt = '-X %s ' % self.method.upper() - - data = '' - if self.method in ('PUT', 'POST', 'PATCH'): - #: -d/--data - send specified data in post request. - if isinstance(self.data, (list, tuple)): - data = data.join(['-d ' + key + '=' + value + ' ' for key, value in self.data]) - elif self._enc_data is not None: - data = '-d ' + self._enc_data + ' ' - - #: Params handled in _build_url - return curl_cmd + method_opt + data + header + '"' + self._build_url() + '"' - class Response(object): """The core :class:`Response ` object. All diff --git a/requests/utils.py b/requests/utils.py index 2b55a164..51ec4bfe 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -165,4 +165,39 @@ def decode_gzip(content): :param content: bytestring to gzip-decode. """ - return zlib.decompress(content, 16+zlib.MAX_WBITS) \ No newline at end of file + return zlib.decompress(content, 16+zlib.MAX_WBITS) + + +def curl_from_request(request): + """Creates a curl command from the request.""" + + #TODO - Auth with User names and accounts + #TODO - Files + #TODO - OAuth + #TODO - Cookies? + + #: -L/--location - if there is a redirect, redo request on the new place + curl_cmd = 'curl -L ' + + #: -H/--header - Extra header to use when getting a web page + header = '' + if request.headers: + header = header.join(['-H "' + key + ':' + value + '" ' for key, value in request.headers.iteritems()]) + + if request.method.upper() == 'HEAD': + #: -I/--head - fetch headers only + method_opt = '-I ' + else: + #: -X/--request - specify request method + method_opt = '-X %s ' % request.method.upper() + + data = '' + if request.method in ('PUT', 'POST', 'PATCH'): + #: -d/--data - send specified data in post request. + if isinstance(request.data, (list, tuple)): + data = data.join(['-d ' + key + '=' + value + ' ' for key, value in request.data]) + elif request._enc_data is not None: + data = '-d ' + request._enc_data + ' ' + + #: Params handled in _build_url + return curl_cmd + method_opt + data + header + '"' + request._build_url() + '"' \ No newline at end of file From 2dfcc0bb7b7eb9060e523f785845919d277ab412 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 24 Aug 2011 00:04:18 -0400 Subject: [PATCH 012/255] string formatting #139 --- requests/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 51ec4bfe..58e7850e 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -195,9 +195,9 @@ def curl_from_request(request): if request.method in ('PUT', 'POST', 'PATCH'): #: -d/--data - send specified data in post request. if isinstance(request.data, (list, tuple)): - data = data.join(['-d ' + key + '=' + value + ' ' for key, value in request.data]) + data = data.join(['-d %s=%s ' % (k, v) for (k, v) in request.data]) elif request._enc_data is not None: - data = '-d ' + request._enc_data + ' ' + data = '-d %s ' % (request._enc_data) #: Params handled in _build_url return curl_cmd + method_opt + data + header + '"' + request._build_url() + '"' \ No newline at end of file From cc2c54093e97965b38d0ae1611bb19cd503d6888 Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Wed, 24 Aug 2011 01:21:44 -0400 Subject: [PATCH 013/255] Adding basic authentication. Also string formatting --- requests/utils.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 58e7850e..8bda43d1 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -171,33 +171,37 @@ def decode_gzip(content): def curl_from_request(request): """Creates a curl command from the request.""" - #TODO - Auth with User names and accounts #TODO - Files - #TODO - OAuth - #TODO - Cookies? + #TODO - OAuth/Other Auths + #TODO - Cookies #: -L/--location - if there is a redirect, redo request on the new place - curl_cmd = 'curl -L ' + curl = 'curl -L ' + + #: -u/--user - Specify the user name and password to use for server auth. + auth = '' + if request.auth is not None: + auth = '-u "%s:%s" ' % (request.auth.username, request.auth.password) + + if request.method.upper() == 'HEAD': + #: -I/--head - fetch headers only + method = '-I ' + else: + #: -X/--request - specify request method + method = '-X %s ' % request.method.upper() #: -H/--header - Extra header to use when getting a web page header = '' if request.headers: - header = header.join(['-H "' + key + ':' + value + '" ' for key, value in request.headers.iteritems()]) - - if request.method.upper() == 'HEAD': - #: -I/--head - fetch headers only - method_opt = '-I ' - else: - #: -X/--request - specify request method - method_opt = '-X %s ' % request.method.upper() + header = header.join(['-H "%s:%s" ' % (k, v) for k, v in request.headers.iteritems()]) data = '' if request.method in ('PUT', 'POST', 'PATCH'): #: -d/--data - send specified data in post request. if isinstance(request.data, (list, tuple)): - data = data.join(['-d %s=%s ' % (k, v) for (k, v) in request.data]) + data = data.join(['-d "%s=%s" ' % (k, v) for (k, v) in request.data]) elif request._enc_data is not None: data = '-d %s ' % (request._enc_data) #: Params handled in _build_url - return curl_cmd + method_opt + data + header + '"' + request._build_url() + '"' \ No newline at end of file + return curl + auth + method + header + data + '"' + request._build_url() + '"' From 3dd2235ac5343ce0c4d4a4b7937f8ce2127f0612 Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Wed, 24 Aug 2011 01:30:38 -0400 Subject: [PATCH 014/255] Adding basic authentication. Also string formatting #139 --- requests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/utils.py b/requests/utils.py index 8bda43d1..461582cf 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -172,7 +172,7 @@ def curl_from_request(request): """Creates a curl command from the request.""" #TODO - Files - #TODO - OAuth/Other Auths + #TODO - OAuth #TODO - Cookies #: -L/--location - if there is a redirect, redo request on the new place From cfee8308772a3c931539688f9928ca6ff81d36cf Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Wed, 24 Aug 2011 01:39:43 -0400 Subject: [PATCH 015/255] Renaming vars. String formatting #139 --- requests/utils.py | 57 ++++++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 461582cf..65ce4f72 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -168,40 +168,41 @@ def decode_gzip(content): return zlib.decompress(content, 16+zlib.MAX_WBITS) -def curl_from_request(request): - """Creates a curl command from the request.""" +def curl_from_request(request): + """Creates a curl command from the request.""" - #TODO - Files - #TODO - OAuth - #TODO - Cookies + #TODO - Files + #TODO - OAuth + #TODO - Cookies - #: -L/--location - if there is a redirect, redo request on the new place - curl = 'curl -L ' + #: -L/--location - if there is a redirect, redo request on the new place + curl = 'curl -L ' #: -u/--user - Specify the user name and password to use for server auth. - auth = '' - if request.auth is not None: + auth = '' + if request.auth is not None: auth = '-u "%s:%s" ' % (request.auth.username, request.auth.password) - if request.method.upper() == 'HEAD': - #: -I/--head - fetch headers only - method = '-I ' - else: - #: -X/--request - specify request method - method = '-X %s ' % request.method.upper() + if request.method.upper() == 'HEAD': + #: -I/--head - fetch headers only + method = '-I ' + else: + #: -X/--request - specify request method + method = '-X %s ' % request.method.upper() - #: -H/--header - Extra header to use when getting a web page - header = '' - if request.headers: - header = header.join(['-H "%s:%s" ' % (k, v) for k, v in request.headers.iteritems()]) + #: -H/--header - Extra header to use when getting a web page + header = '' + if request.headers: + header = header.join(['-H "%s:%s" ' % (k, v) for k, v in request.headers.iteritems()]) - data = '' - if request.method in ('PUT', 'POST', 'PATCH'): - #: -d/--data - send specified data in post request. - if isinstance(request.data, (list, tuple)): - data = data.join(['-d "%s=%s" ' % (k, v) for (k, v) in request.data]) - elif request._enc_data is not None: - data = '-d %s ' % (request._enc_data) + data = '' + if request.method in ('PUT', 'POST', 'PATCH'): + #: -d/--data - send specified data in post request. + if isinstance(request.data, (list, tuple)): + data = data.join(['-d "%s=%s" ' % (k, v) for (k, v) in request.data]) + elif request._enc_data is not None: + data = '-d %s ' % (request._enc_data) - #: Params handled in _build_url - return curl + auth + method + header + data + '"' + request._build_url() + '"' + + #: Params handled in _build_url + return curl + auth + method + header + data + '"' + request._build_url() + '"' From ec0b708b1856be4803ab10eeb50824f739baf0ce Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Wed, 24 Aug 2011 01:40:44 -0400 Subject: [PATCH 016/255] Removing trailing whitespace #139 --- requests/utils.py | 60 +++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 65ce4f72..3f4d9a44 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -168,41 +168,41 @@ def decode_gzip(content): return zlib.decompress(content, 16+zlib.MAX_WBITS) -def curl_from_request(request): - """Creates a curl command from the request.""" +def curl_from_request(request): + """Creates a curl command from the request.""" - #TODO - Files - #TODO - OAuth - #TODO - Cookies + #TODO - Files + #TODO - OAuth + #TODO - Cookies - #: -L/--location - if there is a redirect, redo request on the new place - curl = 'curl -L ' + #: -L/--location - if there is a redirect, redo request on the new place + curl = 'curl -L ' - #: -u/--user - Specify the user name and password to use for server auth. - auth = '' - if request.auth is not None: - auth = '-u "%s:%s" ' % (request.auth.username, request.auth.password) + #: -u/--user - Specify the user name and password to use for server auth. + auth = '' + if request.auth is not None: + auth = '-u "%s:%s" ' % (request.auth.username, request.auth.password) - if request.method.upper() == 'HEAD': - #: -I/--head - fetch headers only - method = '-I ' - else: - #: -X/--request - specify request method - method = '-X %s ' % request.method.upper() + if request.method.upper() == 'HEAD': + #: -I/--head - fetch headers only + method = '-I ' + else: + #: -X/--request - specify request method + method = '-X %s ' % request.method.upper() - #: -H/--header - Extra header to use when getting a web page - header = '' - if request.headers: - header = header.join(['-H "%s:%s" ' % (k, v) for k, v in request.headers.iteritems()]) + #: -H/--header - Extra header to use when getting a web page + header = '' + if request.headers: + header = header.join(['-H "%s:%s" ' % (k, v) for k, v in request.headers.iteritems()]) - data = '' - if request.method in ('PUT', 'POST', 'PATCH'): - #: -d/--data - send specified data in post request. - if isinstance(request.data, (list, tuple)): - data = data.join(['-d "%s=%s" ' % (k, v) for (k, v) in request.data]) - elif request._enc_data is not None: - data = '-d %s ' % (request._enc_data) + data = '' + if request.method in ('PUT', 'POST', 'PATCH'): + #: -d/--data - send specified data in post request. + if isinstance(request.data, (list, tuple)): + data = data.join(['-d "%s=%s" ' % (k, v) for (k, v) in request.data]) + elif request._enc_data is not None: + data = '-d %s ' % (request._enc_data) - #: Params handled in _build_url - return curl + auth + method + header + data + '"' + request._build_url() + '"' + #: Params handled in _build_url + return curl + auth + method + header + data + '"' + request._build_url() + '"' From 71195bb219ed24717961a3f968a00be7f3d8a4a7 Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Wed, 24 Aug 2011 01:50:06 -0400 Subject: [PATCH 017/255] Adding myself to Authors. Small cleanup #139 --- AUTHORS | 1 + requests/utils.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 1eabf651..5a80c622 100644 --- a/AUTHORS +++ b/AUTHORS @@ -42,3 +42,4 @@ Patches and Suggestions - Alejandro Giacometti - Rick Mak - Johan Bergström +- Mike Waldner diff --git a/requests/utils.py b/requests/utils.py index 3f4d9a44..b4e760a9 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -203,6 +203,5 @@ def curl_from_request(request): elif request._enc_data is not None: data = '-d %s ' % (request._enc_data) - #: Params handled in _build_url return curl + auth + method + header + data + '"' + request._build_url() + '"' From 7fa4fdacc9d3e0a418bf2dee1a897bfa799748e9 Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Wed, 24 Aug 2011 22:01:06 -0400 Subject: [PATCH 018/255] Adding in file support #139 --- requests/utils.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index b4e760a9..03e5fac4 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -171,7 +171,6 @@ def decode_gzip(content): def curl_from_request(request): """Creates a curl command from the request.""" - #TODO - Files #TODO - OAuth #TODO - Cookies @@ -179,6 +178,7 @@ def curl_from_request(request): curl = 'curl -L ' #: -u/--user - Specify the user name and password to use for server auth. + #: Basic Auth only for now auth = '' if request.auth is not None: auth = '-u "%s:%s" ' % (request.auth.username, request.auth.password) @@ -195,13 +195,26 @@ def curl_from_request(request): if request.headers: header = header.join(['-H "%s:%s" ' % (k, v) for k, v in request.headers.iteritems()]) - data = '' + form = '' if request.method in ('PUT', 'POST', 'PATCH'): - #: -d/--data - send specified data in post request. - if isinstance(request.data, (list, tuple)): - data = data.join(['-d "%s=%s" ' % (k, v) for (k, v) in request.data]) - elif request._enc_data is not None: - data = '-d %s ' % (request._enc_data) + #: request.files is updated with request.data if both exist. + #: ContentType multipart/form-data is used + if request.files: + #: -F/--form - Emulate form data. To force 'content' to a file, prefix file name @. + for k, v in request.files.iteritems(): + if isinstance(v, file): + form = form + '-F "%s=@%s" ' % (k, v.name) + elif v not in (None, ''): + form = form + '-F "%s=%s" ' % (k, v) + + #: content-type application/x-www-form-urlencoded is used here + else: + #: -d/--data - send specified data in post request. + if isinstance(request.data, (list, tuple)): + form = form.join(['-d "%s=%s" ' % (k, v) for k, v in request.data]) + elif request._enc_data not in (None, ''): + form = "-d '%s' " % (request._enc_data) #: Params handled in _build_url - return curl + auth + method + header + data + '"' + request._build_url() + '"' + return curl + auth + method + header + form + '"' + request._build_url() + '"' + From c243d0939828423bc5af92b07a4026ee406466ee Mon Sep 17 00:00:00 2001 From: Luca De Vitis Date: Thu, 25 Aug 2011 16:17:38 +0200 Subject: [PATCH 019/255] Added gracefull_hooks, default_hooks and 'Accept-Encoding' base_header --- requests/config.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/requests/config.py b/requests/config.py index a92e1f57..e6adfca6 100644 --- a/requests/config.py +++ b/requests/config.py @@ -56,15 +56,25 @@ class Settings(object): settings = Settings() -settings.base_headers = {'User-Agent': 'python-requests.org'} +settings.base_headers = { + 'User-Agent': 'python-requests.org', + 'Accept-Encoding': ', '.join([ 'identity', 'deflate', 'compress', 'gzip' ]), +} settings.accept_gzip = True settings.proxies = None settings.verbose = None settings.timeout = None settings.max_redirects = 30 -# settings.decode_unicode = True -settings.unicode_response = True -settings.decode_response = True +settings.decode_unicode = False +settings.gracefull_hooks = True + +#: A dictionary of default hooks to be applied, based on settings. +settings.default_hooks = { + 'args': list(), + 'pre_request': list(), + 'post_request': list(), + 'response': list() +} #: Use socket.setdefaulttimeout() as fallback? settings.timeout_fallback = True From d8fb6d2a5a169d1bbfd0c1a8cc583a55ecd4e88b Mon Sep 17 00:00:00 2001 From: Luca De Vitis Date: Thu, 25 Aug 2011 16:19:19 +0200 Subject: [PATCH 020/255] No more sets in default hooks, code cleanup, more docs and comments. --- requests/hooks/__init__.py | 69 ++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/requests/hooks/__init__.py b/requests/hooks/__init__.py index 8f7c6b1c..730e6bb3 100644 --- a/requests/hooks/__init__.py +++ b/requests/hooks/__init__.py @@ -25,23 +25,49 @@ Available hooks: import warnings from collections import Iterable from .. import config -from response import unicode_response, decode_response +from . import args +from . import pre_request +from . import post_request +from . import response -def setup_hooks(hooks): - """Setup hooks as a dictionary. Each value is a set of hooks.""" +def setup_hooks(supplied): + """Setup the supplied dictionary of hooks. + Each value is a list of hooks and will extend **default_hooks**. - for key, values in hooks.items(): + :param supplied: a dictionary of hooks. Each value can either be a callable + or a list of callables. + :type supplied: dict + :returns: a dictionary of hooks that extends the **default_hooks** dictionary. + :rtype: dict + """ + + # Copy the default hooks settings. + dispatching = dict([(k, v[:]) for k, v in config.settings.default_hooks]) + + # I abandoned the idea of a dictionary of sets because sets may not keep + # insertion order, while it may be important. Also, there is no real reason + # to force hooks to run once. + for hooks, values in supplied.items(): hook_list = values if isinstance(values, Iterable) else [values] - hooks[key] = set(hook_list) + dispatching[hooks].extends(hook_list) - # Also, based on settings, - if config.settings.unicode_response: - hooks.setdefault('response', set()).add(unicode_response) - if config.settings.decode_response: - hooks.setdefault('response', set()).add(decode_response) - return hooks + # If header is set, maybe response is encoded. Whatever hook you want to + # run on response, content decoding should be first. + if config.settings.base_headers.get('Accept-Encoding', ''): + dispatching['response'].insert(0, response.decode_encoding) -def dispatch_hooks(hooks, hook_data): + if config.settings.decode_unicode: + try: + # Try unicode encoding just after content decoding... + index = dispatching['response'].index(response.decode_encoding) + 1 + except ValueError: + # ... Or as first hook + index = 0 + dispatching['response'].insert(index, response.decode_unicode) + + return dispatching + +def dispatch_hooks(hooks, data): """Dispatches multiple hooks on a given piece of data. :param key: the hooks group to lookup @@ -49,13 +75,22 @@ def dispatch_hooks(hooks, hook_data): :param hooks: the hooks dictionary. The value of each key can be a callable object, or a list of callable objects. :type hooks: dict - :param hook_data: the object on witch the hooks should be applied - :type hook_data: object + :param data: the object on witch the hooks should be applied + :type data: object """ for hook in hooks: try: - # hook must be a callable - hook_data = hook(hook_data) + # hook must be a callable. + data = hook(data) + except Exception, why: + + # Letting users to choose a policy may be an idea. It can be as + # simple as "be gracefull, or not": + # + # config.settings.gracefull_hooks = True | False + if not config.settings.gracefull_hooks: raise + warnings.warn(str(why)) - return hook_data + + return data From 335bf7018d7b073ff9fe3ab68fa92dd12a4c57b8 Mon Sep 17 00:00:00 2001 From: Luca De Vitis Date: Thu, 25 Aug 2011 16:24:55 +0200 Subject: [PATCH 021/255] default hooks copy fixed --- requests/hooks/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requests/hooks/__init__.py b/requests/hooks/__init__.py index 730e6bb3..e737bffb 100644 --- a/requests/hooks/__init__.py +++ b/requests/hooks/__init__.py @@ -42,7 +42,8 @@ def setup_hooks(supplied): """ # Copy the default hooks settings. - dispatching = dict([(k, v[:]) for k, v in config.settings.default_hooks]) + default = config.settings.default_hooks + dispatching = dict([(k, v[:]) for k, v in default.items()]) # I abandoned the idea of a dictionary of sets because sets may not keep # insertion order, while it may be important. Also, there is no real reason From 3bf6c63d522fa096bad1bf4873bc1a9accc8ed61 Mon Sep 17 00:00:00 2001 From: Luca De Vitis Date: Thu, 25 Aug 2011 16:26:10 +0200 Subject: [PATCH 022/255] Applied new hooks api --- requests/api.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/requests/api.py b/requests/api.py index 9c78c17f..cc64fc41 100644 --- a/requests/api.py +++ b/requests/api.py @@ -43,9 +43,7 @@ def request(method, url, method = str(method).upper() - cookies = cookiejar_from_dict(cookies or dict()) - - hooks = setup_hooks(hooks or dict()) + cookies = cookiejar_from_dict(cookies if cookies is not None else dict()) args = dict( method = method, @@ -61,23 +59,24 @@ def request(method, url, proxies = proxies or config.settings.proxies, ) + hooks = setup_hooks(hooks if hooks is not None else dict()) # Arguments manipulation hook. - args = dispatch_hooks(hooks.get('args', []), args) + args = dispatch_hooks(hooks['args'], args) r = Request(**args) # Pre-request hook. - r = dispatch_hooks(hooks.get('pre_request', []), r) + r = dispatch_hooks(hooks['pre_request'], r) # Send the HTTP Request. r.send() # Post-request hook. - r = dispatch_hooks(hooks.get('post_request', []), hooks, r) + r = dispatch_hooks(hooks['post_request'], r) # Response manipulation hook. - r.response = dispatch_hooks(hooks.get('response', []), r.response) + r.response = dispatch_hooks(hooks['response'], r.response) return r.response From d8c923c3850a4f29a39f7da46a77169d8cfe9537 Mon Sep 17 00:00:00 2001 From: Luca De Vitis Date: Thu, 25 Aug 2011 16:28:47 +0200 Subject: [PATCH 023/255] Added json_content and etree_content, more docs and comments. --- requests/hooks/response.py | 83 ++++++++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 21 deletions(-) diff --git a/requests/hooks/response.py b/requests/hooks/response.py index e9ce39b3..beec248b 100644 --- a/requests/hooks/response.py +++ b/requests/hooks/response.py @@ -4,41 +4,82 @@ request.hooks.response This module provide a collection of response hooks. """ -from functools import wraps import zlib +import bz2 from cgi import parse_header +try: + import json +except ImportError: + try: + import simplejson as json + except ImportError: + json = False +try: + from lxml import etree +except ImportError: + try: + import xml.etree.cElementTree as etree + except ImportError: + try: + import xml.etree.ElementTree as etree + except ImportError: + try: + import cElementTree as etree + except ImportError: + try: + import elementtree.ElementTree as etree + except ImportError: + etree = False + #: Dictionary of content decoders. -decoders = { +content_decoders = { # No decoding applied. - 'identity': lambda r: r.content, + 'identity': lambda content: content, # Decode Response file object compressed with deflate. - 'deflate': lambda r: zlib.decompress(r.content), + 'deflate': lambda content: zlib.decompress(content), # Decode Response file object compressed with gzip. - 'gzip': lambda r: zlib.decompress(r.content, 16+zlib.MAX_WBITS), + 'gzip': lambda content: zlib.decompress(content, 16+zlib.MAX_WBITS), + # Decode Response file object compressed with bz2. + # Not a standard Content-Encoding value, but.. + 'bzip2': lambda content: bz2.decompress(content), } # Decode Response file object compressed with compress. -decoders['compress'] = decoders['deflate'] +content_decoders['compress'] = content_decoders['deflate'] -try: - import bz2 -except ImportError: - pass -else: - # Decode Response file object compressed with bz2. - decoders['bzip2'] = lambda r: bz2.decompress(r.content) - -def unicode_response(r): - """Encode response file object in unicode.""" - content_type, params = parse_header(r.headers.get('content-type')) +def decode_unicode(r): + """Encode a :py:class:`requests.models.Response` file object in unicode.""" + content_type, params = parse_header(r.headers['content-type']) charset = params.get('charset', '').strip("'\"") r._content = unicode(r.content, charset) if charset else unicode(r.content) return r -def decode_response(r): - """Decode compressed response content using Contetn-Encoding header.""" - encoding = r.headers.get('content-encoding') - r._content = decoders.get(encoding)(r) +def decode_encoding(r): + """ + Decode a :py:class:`requests.models.Response` content using + Contetn-Encoding header. + """ + # Apply decoding only if the header is set. + encoding = r.headers['content-encoding'] + if encoding: + r._content = content_decoders[encoding](r.content) return r +if json: + def json_content(r): + """ + Turns :py:class:`requests.models.Response` content into a dumped + JSON structure. + """ + r._content = json.dumps(r.content) + return r + +if etree: + def etree_content(r): + """ + Turns :py:class:`requests.models.Response` content into an + ElementTree structure. + """ + r._content = etree.fromstring(r.content) + return r From 3d448f843e82f8c35613ea3d5657e8215d068163 Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Thu, 25 Aug 2011 23:55:24 -0400 Subject: [PATCH 024/255] Adding cookies #139 --- requests/utils.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 03e5fac4..8a0967b9 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -172,7 +172,6 @@ def curl_from_request(request): """Creates a curl command from the request.""" #TODO - OAuth - #TODO - Cookies #: -L/--location - if there is a redirect, redo request on the new place curl = 'curl -L ' @@ -183,6 +182,7 @@ def curl_from_request(request): if request.auth is not None: auth = '-u "%s:%s" ' % (request.auth.username, request.auth.password) + method = '' if request.method.upper() == 'HEAD': #: -I/--head - fetch headers only method = '-I ' @@ -190,6 +190,13 @@ def curl_from_request(request): #: -X/--request - specify request method method = '-X %s ' % request.method.upper() + #: -b/--cookie + #: (HTTP) Pass the data to the HTTP server as a cookie. It is supposedly the + #: data previously received from the server in a "Set-Cookie:" line. + cookies = '' + if request.cookiejar: + cookies = cookies.join(['-b "%s=%s" ' % (k.name, k.value) for k in request.cookiejar]) + #: -H/--header - Extra header to use when getting a web page header = '' if request.headers: @@ -216,5 +223,5 @@ def curl_from_request(request): form = "-d '%s' " % (request._enc_data) #: Params handled in _build_url - return curl + auth + method + header + form + '"' + request._build_url() + '"' + return curl + auth + method + header + cookies + form + '"' + request._build_url() + '"' From 3f98034c2aa5d04e45670ef968c0c7026c188ed9 Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Thu, 25 Aug 2011 23:57:56 -0400 Subject: [PATCH 025/255] Trailing whitespace #139 --- requests/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 8a0967b9..8fa9d659 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -192,10 +192,10 @@ def curl_from_request(request): #: -b/--cookie #: (HTTP) Pass the data to the HTTP server as a cookie. It is supposedly the - #: data previously received from the server in a "Set-Cookie:" line. + #: data previously received from the server in a "Set-Cookie:" line. cookies = '' if request.cookiejar: - cookies = cookies.join(['-b "%s=%s" ' % (k.name, k.value) for k in request.cookiejar]) + cookies = cookies.join(['-b "%s=%s" ' % (k.name, k.value) for k in request.cookiejar]) #: -H/--header - Extra header to use when getting a web page header = '' From 58d63d55bc0cbbb1d543a4165f2e80a808c5577e Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Sun, 28 Aug 2011 21:19:51 -0400 Subject: [PATCH 026/255] Sphinx Documentation #139 --- docs/api.rst | 4 ++++ requests/utils.py | 26 ++++++++++++++++++-------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index e3338b66..328e9e8b 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -48,6 +48,10 @@ Cookies .. autofunction:: cookiejar_from_dict .. autofunction:: add_dict_to_cookiejar +Curl +~~~~ +.. autofunction:: curl_from_request + Encodings ~~~~~~~~~ diff --git a/requests/utils.py b/requests/utils.py index 8fa9d659..d4f05b5e 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -169,11 +169,21 @@ def decode_gzip(content): def curl_from_request(request): - """Creates a curl command from the request.""" + """Returns a curl command from the request. + + :param request: The :class:`Request ` object + + Example: + | import requests + | from requests.utils import curl_from_request + | r = requests.get('http://httpbin.org/get') + | curl_from_request(r.request) + + """ #TODO - OAuth - #: -L/--location - if there is a redirect, redo request on the new place + #: -L/--location - if there is a redirect, redo request on the new place. curl = 'curl -L ' #: -u/--user - Specify the user name and password to use for server auth. @@ -184,10 +194,10 @@ def curl_from_request(request): method = '' if request.method.upper() == 'HEAD': - #: -I/--head - fetch headers only + #: -I/--head - fetch headers only. method = '-I ' else: - #: -X/--request - specify request method + #: -X/--request - specify request method. method = '-X %s ' % request.method.upper() #: -b/--cookie @@ -197,15 +207,15 @@ def curl_from_request(request): if request.cookiejar: cookies = cookies.join(['-b "%s=%s" ' % (k.name, k.value) for k in request.cookiejar]) - #: -H/--header - Extra header to use when getting a web page + #: -H/--header - Extra header to use when getting a web page. header = '' if request.headers: header = header.join(['-H "%s:%s" ' % (k, v) for k, v in request.headers.iteritems()]) form = '' if request.method in ('PUT', 'POST', 'PATCH'): - #: request.files is updated with request.data if both exist. - #: ContentType multipart/form-data is used + #: request.files is updated with request.data if both exist, so only iterate request.files. + #: ContentType multipart/form-data is used. if request.files: #: -F/--form - Emulate form data. To force 'content' to a file, prefix file name @. for k, v in request.files.iteritems(): @@ -214,7 +224,7 @@ def curl_from_request(request): elif v not in (None, ''): form = form + '-F "%s=%s" ' % (k, v) - #: content-type application/x-www-form-urlencoded is used here + #: content-type application/x-www-form-urlencoded is used here. else: #: -d/--data - send specified data in post request. if isinstance(request.data, (list, tuple)): From e46351cfbd1ee908fecdd54c52bd928c370b4c05 Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Sun, 28 Aug 2011 23:31:43 -0400 Subject: [PATCH 027/255] Unit Tests for curl_from_request #139 --- test_requests.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/test_requests.py b/test_requests.py index dd923471..b3f590cd 100755 --- a/test_requests.py +++ b/test_requests.py @@ -14,6 +14,7 @@ except ImportError: import requests from requests.sessions import Session +from requests.utils import curl_from_request HTTPBIN_URL = 'http://httpbin.org/' @@ -470,6 +471,53 @@ class RequestsTestSuite(unittest.TestCase): self.assertEqual(r2.status_code, 200) + def test_curl_HTTP_OK_GET(self): + curl_str = 'curl -L -X GET -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" "http://httpbin.org//"' + r = requests.get(httpbin('/')) + self.assertEqual(curl_from_request(r.request), curl_str) + + + def test_curl_HTTP_OK_GET_WITH_PARAMS(self): + curl_str = 'curl -L -X GET -H "Accept-Encoding:gzip" -H "User-agent:Mozilla/5.0" "http://httpbin.org/user-agent"' + + heads = {'User-agent': 'Mozilla/5.0'} + r = requests.get(httpbin('user-agent'), headers=heads) + self.assertEqual(curl_from_request(r.request), curl_str) + + + def test_curl_HTTP_OK_HEAD(self): + curl_str ='curl -L -I -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" "http://httpbin.org//"' + r = requests.head(httpbin('/')) + self.assertEqual(curl_from_request(r.request), curl_str) + + + def test_curl_HTTP_OK_PATCH(self): + curl_str = 'curl -L -X PATCH -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" "http://httpbin.org/patch"' + r = requests.patch(httpbin('patch')) + self.assertEqual(curl_from_request(r.request), curl_str) + + + def test_curl_AUTH_HTTPS_OK_GET(self): + curl_str = 'curl -L -u "user:pass" -X GET -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" "https://httpbin.ep.io/basic-auth/user/pass"' + auth = ('user', 'pass') + r = requests.get(httpsbin('basic-auth', 'user', 'pass'), auth=auth) + self.assertEqual(curl_from_request(r.request), curl_str) + + + def test_curl_POSTBIN_GET_POST_FILES(self): + curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -d "some=data" "http://httpbin.org/post"' + post = requests.post(httpbin('post'), data={'some': 'data'}) + self.assertEqual(curl_from_request(post.request), curl_str) + + curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -F "some=@test_requests.py" "https://httpbin.ep.io/post"' + post2 = requests.post(httpsbin('post'), files={'some': open('test_requests.py')}) + self.assertEqual(curl_from_request(post2.request), curl_str) + + #TODO - This doesn't seem right with \ \ + #curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -d \'[{"some": "json"}]\' "http://httpbin.org/post"' + #post3 = requests.post(httpbin('post'), data='[{"some": "json"}]') + #self.assertEqual(curl_from_request(post3.request), curl_str) + if __name__ == '__main__': unittest.main() From 07dd324e2f78ee208e35d3faa963ebab237ff95f Mon Sep 17 00:00:00 2001 From: Luca De Vitis Date: Thu, 1 Sep 2011 18:22:12 +0200 Subject: [PATCH 028/255] Reverted hooks from sub-package to module. --- requests/hooks.py | 184 +++++++++++++++++++++++++++++++++ requests/hooks/__init__.py | 97 ----------------- requests/hooks/args.py | 7 -- requests/hooks/post_request.py | 7 -- requests/hooks/pre_request.py | 7 -- requests/hooks/response.py | 85 --------------- 6 files changed, 184 insertions(+), 203 deletions(-) create mode 100644 requests/hooks.py delete mode 100644 requests/hooks/__init__.py delete mode 100644 requests/hooks/args.py delete mode 100644 requests/hooks/post_request.py delete mode 100644 requests/hooks/pre_request.py delete mode 100644 requests/hooks/response.py diff --git a/requests/hooks.py b/requests/hooks.py new file mode 100644 index 00000000..e2980ac3 --- /dev/null +++ b/requests/hooks.py @@ -0,0 +1,184 @@ +# -*- coding: utf-8 -*- + +""" +requests.hooks +~~~~~~~~~~~~~~ + +This module provides the capabilities for the Requests hooks system. + +Available hooks: + +``args``: + A dictionary of the arguments being sent to Request(). + +``pre_request``: + The Request object, directly before being sent. + +``post_request``: + The Request object, directly after being sent. + +``response``: + The response generated from a Request. + +""" + +import warnings +from collections import Iterable +import config +import zlib +import bz2 +from cgi import parse_header + +def setup_hooks(supplied): + """Setup a hooks mapping, based on the supplied argument. Eache mapping + value will be list of hooks that will extend the **default_hooks**. + + :param supplied: a dictionary of hooks. Each value can either be a callable + or a list of callables. + :type supplied: dict + :returns: a dictionary of hooks that extends the **default_hooks** dictionary. + :rtype: dict + """ + + # Copy the default hooks settings. + default = config.settings.default_hooks + dispatching = dict([(k, v[:]) for k, v in default.items()]) + + # I abandoned the idea of a dictionary of sets because sets may not keep + # insertion order, while it may be important. Also, there is no real reason + # to force hooks to run once. + for hooks, values in supplied.items(): + hook_list = values if isinstance(values, Iterable) else [values] + dispatching[hooks].extend(hook_list) + + # If header is set by config, maybe response is encoded. + if config.settings.base_headers.get('Accept-Encoding', ''): + if not decode_encoding in dispatching['response']: + # It's safer to put decoding as first hook. + dispatching['response'].insert(0, decode_encoding) + + if config.settings.decode_unicode: + try: + # Try unicode encoding just after content decoding... + index = dispatching['response'].index(decode_encoding) + 1 + except ValueError: + # ... Or as first hook + index = 0 + dispatching['response'].insert(index, decode_unicode) + + return dispatching + +def dispatch_hooks(hooks, data): + """Dispatches multiple hooks on a given piece of data. + + :param key: the hooks group to lookup + :type key: str + :param hooks: the hooks dictionary. The value of each key can be a callable + object, or a list of callable objects. + :type hooks: dict + :param data: the object on witch the hooks should be applied + :type data: object + """ + for hook in hooks: + try: + # hook must be a callable. + data = hook(data) + + except Exception, why: + + # Letting users to choose a policy may be an idea. It can be as + # simple as "be gracefull, or not": + # + # config.settings.gracefull_hooks = True | False + if not config.settings.gracefull_hooks: raise + + warnings.warn(str(why)) + + return data + +#: Example response hook that turns a JSON formatted +#: :py:class:`requests.models.Response.content` into a dumped data structure:: +#: +#: try: +#: import json +#: except ImportError: +#: try: +#: import simplejson as json +#: except ImportError: +#: json = False +#: +#: if json: +#: def json_content(r): +#: """Turns content into a dumped JSON structure.""" +#: r._content = json.dumps(r.content) +#: return r +#: +#: Example response hook that turns an XML formatted +#: :py:class:`requests.models.Response.content` into an ElementTree:: +#: +#: try: +#: from lxml import etree +#: except ImportError: +#: try: +#: import xml.etree.cElementTree as etree +#: except ImportError: +#: try: +#: import xml.etree.ElementTree as etree +#: except ImportError: +#: try: +#: import cElementTree as etree +#: except ImportError: +#: try: +#: import elementtree.ElementTree as etree +#: except ImportError: +#: etree = False +#: +#: if etree: +#: def etree_content(r): +#: """Turns content into an ElementTree structure.""" +#: r._content = etree.fromstring(r.content) +#: return r + +def decode_unicode(r): + """Encode content into unicode string. + + :param r: response object + :type r: :py:class:`requests.models.Response` + :returns: the same input object. + :rtype: :py:class:`requests.models.Response` + """ + content_type, params = parse_header(r.headers['content-type']) + charset = params.get('charset', '').strip("'\"") + r._content = unicode(r.content, charset) if charset else unicode(r.content) + return r + +def decode_encoding(r): + """Decode content using Contetn-Encoding header. + + :param r: response object + :type r: :py:class:`requests.models.Response` + :returns: the same input object. + :rtype: :py:class:`requests.models.Response` + """ + + # Dictionary of content decoders. + decode = { + # No decoding applied. + 'identity': lambda content: content, + # Decode Response content compressed with deflate. + 'deflate': lambda content: zlib.decompress(content), + # Decode Response content compressed with gzip. + 'gzip': lambda content: zlib.decompress(content, 16+zlib.MAX_WBITS), + # Decode Response content compressed with bz2. + # Not a standard Content-Encoding value, but.. + 'bzip2': lambda content: bz2.decompress(content), + } + # Decode Response content compressed with compress. + # If I understood zlib... + decode['compress'] = decode['deflate'] + + # Apply decoding only if the header is set. + encoding = r.headers['content-encoding'] + if encoding: + r._content = decode[encoding](r.content) + return r diff --git a/requests/hooks/__init__.py b/requests/hooks/__init__.py deleted file mode 100644 index e737bffb..00000000 --- a/requests/hooks/__init__.py +++ /dev/null @@ -1,97 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -requests.hooks -~~~~~~~~~~~~~~ - -This module provides the capabilities for the Requests hooks system. - -Available hooks: - -``args``: - A dictionary of the arguments being sent to Request(). - -``pre_request``: - The Request object, directly before being sent. - -``post_request``: - The Request object, directly after being sent. - -``response``: - The response generated from a Request. - -""" - -import warnings -from collections import Iterable -from .. import config -from . import args -from . import pre_request -from . import post_request -from . import response - -def setup_hooks(supplied): - """Setup the supplied dictionary of hooks. - Each value is a list of hooks and will extend **default_hooks**. - - :param supplied: a dictionary of hooks. Each value can either be a callable - or a list of callables. - :type supplied: dict - :returns: a dictionary of hooks that extends the **default_hooks** dictionary. - :rtype: dict - """ - - # Copy the default hooks settings. - default = config.settings.default_hooks - dispatching = dict([(k, v[:]) for k, v in default.items()]) - - # I abandoned the idea of a dictionary of sets because sets may not keep - # insertion order, while it may be important. Also, there is no real reason - # to force hooks to run once. - for hooks, values in supplied.items(): - hook_list = values if isinstance(values, Iterable) else [values] - dispatching[hooks].extends(hook_list) - - # If header is set, maybe response is encoded. Whatever hook you want to - # run on response, content decoding should be first. - if config.settings.base_headers.get('Accept-Encoding', ''): - dispatching['response'].insert(0, response.decode_encoding) - - if config.settings.decode_unicode: - try: - # Try unicode encoding just after content decoding... - index = dispatching['response'].index(response.decode_encoding) + 1 - except ValueError: - # ... Or as first hook - index = 0 - dispatching['response'].insert(index, response.decode_unicode) - - return dispatching - -def dispatch_hooks(hooks, data): - """Dispatches multiple hooks on a given piece of data. - - :param key: the hooks group to lookup - :type key: str - :param hooks: the hooks dictionary. The value of each key can be a callable - object, or a list of callable objects. - :type hooks: dict - :param data: the object on witch the hooks should be applied - :type data: object - """ - for hook in hooks: - try: - # hook must be a callable. - data = hook(data) - - except Exception, why: - - # Letting users to choose a policy may be an idea. It can be as - # simple as "be gracefull, or not": - # - # config.settings.gracefull_hooks = True | False - if not config.settings.gracefull_hooks: raise - - warnings.warn(str(why)) - - return data diff --git a/requests/hooks/args.py b/requests/hooks/args.py deleted file mode 100644 index 6ac6fb6c..00000000 --- a/requests/hooks/args.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -request.hooks.args -~~~~~~~~~~~~~~~~~~ - -This module provide a collection of args hooks. -""" - diff --git a/requests/hooks/post_request.py b/requests/hooks/post_request.py deleted file mode 100644 index 3a4ff543..00000000 --- a/requests/hooks/post_request.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -request.hooks.post_request -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This module provide a collection of post_request hooks. -""" - diff --git a/requests/hooks/pre_request.py b/requests/hooks/pre_request.py deleted file mode 100644 index ca1e715f..00000000 --- a/requests/hooks/pre_request.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -request.hooks.pre_request -~~~~~~~~~~~~~~~~~~~~~~~~~ - -This module provide a collection of pre_request hooks. -""" - diff --git a/requests/hooks/response.py b/requests/hooks/response.py deleted file mode 100644 index beec248b..00000000 --- a/requests/hooks/response.py +++ /dev/null @@ -1,85 +0,0 @@ -""" -request.hooks.response -~~~~~~~~~~~~~~~~~~~~~~ - -This module provide a collection of response hooks. -""" -import zlib -import bz2 -from cgi import parse_header - -try: - import json -except ImportError: - try: - import simplejson as json - except ImportError: - json = False -try: - from lxml import etree -except ImportError: - try: - import xml.etree.cElementTree as etree - except ImportError: - try: - import xml.etree.ElementTree as etree - except ImportError: - try: - import cElementTree as etree - except ImportError: - try: - import elementtree.ElementTree as etree - except ImportError: - etree = False - -#: Dictionary of content decoders. -content_decoders = { - # No decoding applied. - 'identity': lambda content: content, - # Decode Response file object compressed with deflate. - 'deflate': lambda content: zlib.decompress(content), - # Decode Response file object compressed with gzip. - 'gzip': lambda content: zlib.decompress(content, 16+zlib.MAX_WBITS), - # Decode Response file object compressed with bz2. - # Not a standard Content-Encoding value, but.. - 'bzip2': lambda content: bz2.decompress(content), -} - -# Decode Response file object compressed with compress. -content_decoders['compress'] = content_decoders['deflate'] - -def decode_unicode(r): - """Encode a :py:class:`requests.models.Response` file object in unicode.""" - content_type, params = parse_header(r.headers['content-type']) - charset = params.get('charset', '').strip("'\"") - r._content = unicode(r.content, charset) if charset else unicode(r.content) - return r - -def decode_encoding(r): - """ - Decode a :py:class:`requests.models.Response` content using - Contetn-Encoding header. - """ - # Apply decoding only if the header is set. - encoding = r.headers['content-encoding'] - if encoding: - r._content = content_decoders[encoding](r.content) - return r - -if json: - def json_content(r): - """ - Turns :py:class:`requests.models.Response` content into a dumped - JSON structure. - """ - r._content = json.dumps(r.content) - return r - -if etree: - def etree_content(r): - """ - Turns :py:class:`requests.models.Response` content into an - ElementTree structure. - """ - r._content = etree.fromstring(r.content) - return r From 824d54f13111387df941f7e6f02b8ac064bec04e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Fri, 2 Sep 2011 15:06:58 +0200 Subject: [PATCH 029/255] Only the path should be encoded (not the query, otherwise it makes a wrong redirection). --- requests/models.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index 2d7fc8fe..4af0597d 100644 --- a/requests/models.py +++ b/requests/models.py @@ -224,8 +224,11 @@ class Request(object): # Facilitate non-RFC2616-compliant 'location' headers # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') - if not urlparse(url).netloc: - url = urljoin(r.url, urllib.quote(urllib.unquote(url))) + parsed_url = urlparse(url) + if not parsed_url.netloc: + parsed_url = list(parsed_url) + parsed_url[2] = urllib.quote(urllib.unquote(parsed_url[2])) + url = urljoin(r.url, str(urlunparse(parsed_url))) # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 if r.status_code is codes.see_other: From 2aa32c16dcb905e303b2bc19ea95d616fba28adf Mon Sep 17 00:00:00 2001 From: Mike Waldner Date: Sat, 3 Sep 2011 20:54:55 -0400 Subject: [PATCH 030/255] Updated unit tests #139 --- test_requests.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test_requests.py b/test_requests.py index b3f590cd..a4484c41 100755 --- a/test_requests.py +++ b/test_requests.py @@ -513,10 +513,9 @@ class RequestsTestSuite(unittest.TestCase): post2 = requests.post(httpsbin('post'), files={'some': open('test_requests.py')}) self.assertEqual(curl_from_request(post2.request), curl_str) - #TODO - This doesn't seem right with \ \ - #curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -d \'[{"some": "json"}]\' "http://httpbin.org/post"' - #post3 = requests.post(httpbin('post'), data='[{"some": "json"}]') - #self.assertEqual(curl_from_request(post3.request), curl_str) + curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -d \'[{"some": "json"}]\' "http://httpbin.org/post"' + post3 = requests.post(httpbin('post'), data='[{"some": "json"}]') + self.assertEqual(curl_from_request(post3.request), curl_str) if __name__ == '__main__': From 724ff7e4fa3af4d18b948c84fec48833006fea0b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 5 Sep 2011 16:25:09 -0400 Subject: [PATCH 031/255] yep --- HISTORY.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 33d56b93..0b129066 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,11 +1,12 @@ History ------- -* Automatic decoding of unicode, when available. +* Automatic decoding of unicode, based on HTTP Headers. * New ``decode_unicode`` setting * Removal of ``r.read/close`` methods -* New ``r.fo`` interface for advanced response usage. - +* New ``r.fo`` interface for advanced response usage.* +* Automatic expansion of parameterized headers +* Turn off Redirects for GET/HEAD requests 0.6.1 (2011-08-20) ++++++++++++++++++ From 8da0abfa1a0b0de121bed615f419b4c9edaf95c8 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 7 Sep 2011 17:52:46 -0400 Subject: [PATCH 032/255] that shouldn't be there... --- setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.py b/setup.py index 6b95c31e..c5ad9d64 100755 --- a/setup.py +++ b/setup.py @@ -22,9 +22,6 @@ if sys.argv[-1] == "test": required = [] -if sys.version_info[:2] < (2,6): - required.append('simplejson') - setup( name='requests', version=requests.__version__, From 46e01db7853ae5d0670598b49dab5055acbbe3fb Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 7 Sep 2011 18:10:09 -0400 Subject: [PATCH 033/255] Makefile! --- Makefile | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..37600f59 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +init: + pip install -r reqs.txt + +test: + python test_requests.py + +site: + # Building Docs + +docs: site + cd docs; make dirhtml \ No newline at end of file From 45fabf4b9642b262e457d52660daefb160be3ad9 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 7 Sep 2011 18:10:58 -0400 Subject: [PATCH 034/255] cleaner setup.py --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index c5ad9d64..de7dcb0e 100755 --- a/setup.py +++ b/setup.py @@ -12,12 +12,12 @@ except ImportError: -if sys.argv[-1] == "publish": - os.system("python setup.py sdist upload") +if 'publish' in sys.argv: + os.system('python setup.py sdist upload') sys.exit() -if sys.argv[-1] == "test": - os.system("python test_requests.py") +if 'test' in sys.argv: + os.system('python test_requests.py') sys.exit() required = [] From fa47067802e89a06eab564547535abe6ba687166 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 7 Sep 2011 18:19:54 -0400 Subject: [PATCH 035/255] hrm --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 37600f59..6ec49602 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,6 @@ test: python test_requests.py site: - # Building Docs + cd docs; make dirhtml docs: site - cd docs; make dirhtml \ No newline at end of file From d5a038463095c03b3fbd4ef6740fa3614f441ac4 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 7 Sep 2011 18:41:07 -0400 Subject: [PATCH 036/255] trailing --- test_requests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_requests.py b/test_requests.py index f554adb1..5baff568 100755 --- a/test_requests.py +++ b/test_requests.py @@ -512,15 +512,15 @@ class RequestsTestSuite(unittest.TestCase): def test_curl_POSTBIN_GET_POST_FILES(self): - curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -d "some=data" "http://httpbin.org/post"' + curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -d "some=data" "http://httpbin.org/post"' post = requests.post(httpbin('post'), data={'some': 'data'}) self.assertEqual(curl_from_request(post.request), curl_str) - curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -F "some=@test_requests.py" "https://httpbin.ep.io/post"' + curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -F "some=@test_requests.py" "https://httpbin.ep.io/post"' post2 = requests.post(httpsbin('post'), files={'some': open('test_requests.py')}) self.assertEqual(curl_from_request(post2.request), curl_str) - curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -d \'[{"some": "json"}]\' "http://httpbin.org/post"' + curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -d \'[{"some": "json"}]\' "http://httpbin.org/post"' post3 = requests.post(httpbin('post'), data='[{"some": "json"}]') self.assertEqual(curl_from_request(post3.request), curl_str) From ccb9eb28e1c45e7b835f6d6fd74ff2e7e1a80eb2 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 10 Sep 2011 19:03:16 -0400 Subject: [PATCH 037/255] slimming down the repo --- HACKING | 15 --------------- requests/config.py | 37 ------------------------------------- tox.ini | 8 -------- 3 files changed, 60 deletions(-) delete mode 100644 HACKING delete mode 100644 tox.ini diff --git a/HACKING b/HACKING deleted file mode 100644 index f23d6fb0..00000000 --- a/HACKING +++ /dev/null @@ -1,15 +0,0 @@ -Where possible, please follow PEP8 with regard to coding style. Sometimes the -line length restriction is too hard to follow, so don't bend over backwards -there. - -Triple-quotes should always be """, single quotes are ' unless using " would -result in less escaping within the string. - -All modules, functions, and methods should be well documented reStructuredText -for Sphinx AutoDoc. - -All functionality should be available in pure Python. Optional C (via Cython) -implementations may be written for performance reasons, but should never -replace the Python implementation. - -Lastly, don't take yourself too seriously :) diff --git a/requests/config.py b/requests/config.py index 794109c5..676a5f0e 100644 --- a/requests/config.py +++ b/requests/config.py @@ -9,48 +9,11 @@ This module provides the Requests settings feature set. """ class Settings(object): - _singleton = {} - - # attributes with defaults - __attrs__ = [] def __init__(self, **kwargs): super(Settings, self).__init__() - self.__dict__ = self._singleton - - - def __call__(self, *args, **kwargs): - # new instance of class to call - r = self.__class__() - - # cache previous settings for __exit__ - r.__cache = self.__dict__.copy() - map(self.__cache.setdefault, self.__attrs__) - - # set new settings - self.__dict__.update(*args, **kwargs) - - return r - - - def __enter__(self): - pass - - - def __exit__(self, *args): - - # restore cached copy - self.__dict__.update(self.__cache.copy()) - del self.__cache - - def __getattribute__(self, key): - if key in object.__getattribute__(self, '__attrs__'): - try: - return object.__getattribute__(self, key) - except AttributeError: - return None return object.__getattribute__(self, key) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index b0fc2138..00000000 --- a/tox.ini +++ /dev/null @@ -1,8 +0,0 @@ -[tox] -envlist = py25,py26,py27 - -[testenv] -commands=py.test --junitxml=junit-{envname}.xml -deps = - pytest - omnijson \ No newline at end of file From 058ef27178b49eb2a97bb5ea40a077248a58e234 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 10 Sep 2011 19:11:03 -0400 Subject: [PATCH 038/255] clarify docs version --- requests/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/core.py b/requests/core.py index c4366647..e1ba1853 100644 --- a/requests/core.py +++ b/requests/core.py @@ -12,8 +12,8 @@ This module implements the main Requests system. """ __title__ = 'requests' -__version__ = '0.6.1' -__build__ = 0x000601 +__version__ = '0.6.2 (dev)' +__build__ = 0x000602 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' From 4df5aadb010d99862f16ea4175a2aef9450bc33b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 10 Sep 2011 19:40:33 -0400 Subject: [PATCH 039/255] ambiguous --- docs/index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 8bef62bb..3021a1bf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,10 +36,10 @@ multipart files, and parameters with simple Python dictionaries, and access the response data in the same way. It's powered by :py:class:`urllib2`, but it does all the hard work and crazy hacks for you. -Testimonals ------------ +Testimonials +------------ -`Twitter, Inc `_ uses Requests internally. +`Twitter, Inc `_ and an Unnamed Federal US Institution use Requests internally. **Daniel Greenfeld** Nuked a 1200 LOC spaghetti code library with 10 lines of code thanks to From d697c37d3e5039cc3ebb7254d879872a7f4dc378 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 10 Sep 2011 19:43:52 -0400 Subject: [PATCH 040/255] readabilty --- docs/index.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 3021a1bf..205b7a3c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -39,7 +39,9 @@ all the hard work and crazy hacks for you. Testimonials ------------ -`Twitter, Inc `_ and an Unnamed Federal US Institution use Requests internally. +`Twitter, Inc `_, +`Readability `_, +and an Unnamed Federal US Institution use Requests internally. **Daniel Greenfeld** Nuked a 1200 LOC spaghetti code library with 10 lines of code thanks to From 37d269c1049c438ac67e5ff5347e0194c22df330 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 10 Sep 2011 19:45:34 -0400 Subject: [PATCH 041/255] more concise --- docs/index.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 205b7a3c..7f3f5927 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -40,8 +40,9 @@ Testimonials ------------ `Twitter, Inc `_, -`Readability `_, -and an Unnamed Federal US Institution use Requests internally. +a Federal US Institution, +and `Readability `_ +use Requests internally. **Daniel Greenfeld** Nuked a 1200 LOC spaghetti code library with 10 lines of code thanks to From 1b2533f92db6d9cf4983db71af9446aa331cdf0c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 10 Sep 2011 19:54:16 -0400 Subject: [PATCH 042/255] better --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 7f3f5927..15f38743 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -40,7 +40,7 @@ Testimonials ------------ `Twitter, Inc `_, -a Federal US Institution, +a U.S. Federal Institution, and `Readability `_ use Requests internally. From c8914bd4d4c7e9e7638cdb43d5a3513315f19028 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 10 Sep 2011 20:18:49 -0400 Subject: [PATCH 043/255] X --- requests/patches.py | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 requests/patches.py diff --git a/requests/patches.py b/requests/patches.py deleted file mode 100644 index 43a3b4c4..00000000 --- a/requests/patches.py +++ /dev/null @@ -1,5 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -requests.monkeys -""" From 34e3c7b68febd3b320544e77037676a3a6f35f63 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 14 Sep 2011 08:34:36 -0400 Subject: [PATCH 044/255] fix for #160 --- AUTHORS | 1 + requests/api.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index ceba4873..af00b19e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -44,3 +44,4 @@ Patches and Suggestions - Johan Bergström - Josselin Jacquard - Mike Waldner +- Serge Domkowski \ No newline at end of file diff --git a/requests/api.py b/requests/api.py index 102bc756..60d0d852 100644 --- a/requests/api.py +++ b/requests/api.py @@ -17,10 +17,10 @@ from .status_codes import codes from .hooks import dispatch_hook from .utils import cookiejar_from_dict, header_expand -from urlparse import urlparse __all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete') + def request(method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=False, proxies=None, hooks=None): @@ -88,7 +88,6 @@ def request(method, url, def get(url, **kwargs): - """Sends a GET request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. @@ -144,7 +143,7 @@ def patch(url, data='', **kwargs): :param **kwargs: Optional arguments that ``request`` takes. """ - return request('patch', url, **kwargs) + return request('patch', url, data=data, **kwargs) def delete(url, **kwargs): From a8e77688f0a610bc8f8d304f2d15094aac149ac7 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 14 Sep 2011 08:39:52 -0400 Subject: [PATCH 045/255] Merge pull request #150 --- requests/models.py | 34 +++++++++++++++++++++++++++++++++- requests/utils.py | 36 ++++++++++++++++++++++++++++++++++++ test_requests.py | 1 + 3 files changed, 70 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 6180e969..a16d32ea 100644 --- a/requests/models.py +++ b/requests/models.py @@ -9,6 +9,7 @@ requests.models import urllib import urllib2 import socket +import codecs import zlib @@ -21,7 +22,7 @@ from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicA from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers -from .utils import dict_from_cookiejar, get_unicode_from_response, decode_gzip +from .utils import dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip from .status_codes import codes from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects @@ -396,6 +397,7 @@ class Response(object): def __init__(self): self._content = None + self._content_consumed = False #: Integer Code of responded HTTP Status. self.status_code = None @@ -438,6 +440,31 @@ class Response(object): return not self.error + def iter_content(self, chunk_size=10 * 1024, decode_unicode=None): + """Iterates over the response data. This avoids reading the content + at once into memory for large responses. The chunk size is the number + of bytes it should read into memory. This is not necessarily the + length of each item returned as decoding can take place. + """ + if self._content_consumed: + raise RuntimeError('The content for this response was ' + 'already consumed') + + def generate(): + while 1: + chunk = self.fo.read(chunk_size) + if not chunk: + break + yield chunk + self._content_consumed = True + gen = generate() + if 'gzip' in self.headers.get('content-encoding', ''): + gen = stream_decode_gzip(gen) + if decode_unicode is None: + decode_unicode = settings.decode_unicode + if decode_unicode: + gen = stream_decode_response_unicode(gen, self) + return gen @property def content(self): @@ -448,6 +475,10 @@ class Response(object): if self._content is not None: return self._content + if self._content_consumed: + raise RuntimeError('The content for this response was ' + 'already consumed') + # Read the contents. self._content = self.fo.read() @@ -462,6 +493,7 @@ class Response(object): if settings.decode_unicode: self._content = get_unicode_from_response(self) + self._content_consumed = True return self._content diff --git a/requests/utils.py b/requests/utils.py index c1aed434..17f79a8a 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -10,6 +10,7 @@ that are also useful for external consumption. """ import cgi +import codecs import cookielib import re import zlib @@ -177,6 +178,24 @@ def unicode_from_html(content): return content +def stream_decode_response_unicode(iterator, r): + """Stream decodes a iterator.""" + encoding = get_encoding_from_headers(r.headers) + if encoding is None: + for item in iterator: + yield item + return + + decoder = codecs.getincrementaldecoder(encoding)(errors='replace') + for chunk in iterator: + rv = decoder.decode(chunk) + if rv: + yield rv + rv = decoder.decode('', final=True) + if rv: + yield rv + + def get_unicode_from_response(r): """Returns the requested content back in unicode. @@ -217,8 +236,25 @@ def decode_gzip(content): """ return zlib.decompress(content, 16+zlib.MAX_WBITS) + return zlib.decompress(content, 16+zlib.MAX_WBITS) + return zlib.decompress(content, 16 + zlib.MAX_WBITS) +def stream_decode_gzip(iterator): + """Stream decodes a gzip-encoded iterator""" + try: + dec = zlib.decompressobj(16 + zlib.MAX_WBITS) + for chunk in iterator: + rv = dec.decompress(chunk) + if rv: + yield rv + buf = dec.decompress('') + rv = buf + dec.flush() + if rv: + yield rv + except zlib.error: + pass + def curl_from_request(request): """Returns a curl command from the request. diff --git a/test_requests.py b/test_requests.py index 5baff568..406d23c1 100755 --- a/test_requests.py +++ b/test_requests.py @@ -478,6 +478,7 @@ class RequestsTestSuite(unittest.TestCase): self.assertEqual(r2.status_code, 200) + def test_curl_HTTP_OK_GET(self): curl_str = 'curl -L -X GET -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" "http://httpbin.org//"' r = requests.get(httpbin('/')) From 2a9ae4b367fc06eeb1c3c24d1c9f741396dd7576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Thu, 15 Sep 2011 11:57:10 +0200 Subject: [PATCH 046/255] Fixed #161 (quoting issue). --- requests/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index a16d32ea..742acf2c 100644 --- a/requests/models.py +++ b/requests/models.py @@ -236,7 +236,7 @@ class Request(object): parsed_url = urlparse(url) if not parsed_url.netloc: parsed_url = list(parsed_url) - parsed_url[2] = urllib.quote(urllib.unquote(parsed_url[2])) + parsed_url[2] = urllib.quote(parsed_url[2], safe="%/:=&?~#+!$,;'@()*[]") url = urljoin(r.url, str(urlunparse(parsed_url))) # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 @@ -290,7 +290,7 @@ class Request(object): netloc = netloc.encode('idna') if isinstance(path, unicode): path = path.encode('utf-8') - path = urllib.quote(urllib.unquote(path)) + path = urllib.quote(path, safe="%/:=&?~#+!$,;'@()*[]") self.url = str(urlunparse([ scheme, netloc, path, params, query, fragment ])) if self._enc_params: From 4b93bb40829a0db2a321fa5a8aba24b0b4982e77 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 15 Sep 2011 10:08:25 -0400 Subject: [PATCH 047/255] history! --- HISTORY.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 0b129066..305fb437 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,6 +7,8 @@ History * New ``r.fo`` interface for advanced response usage.* * Automatic expansion of parameterized headers * Turn off Redirects for GET/HEAD requests +* Fix for Python Pre-2.7 URL quoting +* New ``r.iter_content`` method. 0.6.1 (2011-08-20) ++++++++++++++++++ From 3443ede52a74ea1c3bc0bb9a1d835a106448d0de Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 15 Sep 2011 10:09:36 -0400 Subject: [PATCH 048/255] ref to bug in history --- HISTORY.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 305fb437..5e976e82 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,8 +7,9 @@ History * New ``r.fo`` interface for advanced response usage.* * Automatic expansion of parameterized headers * Turn off Redirects for GET/HEAD requests -* Fix for Python Pre-2.7 URL quoting -* New ``r.iter_content`` method. +* Fix for Python Pre-2.7 URL quoting (http://bugs.python.org/issue918368) +* New ``r.iter_content`` method + 0.6.1 (2011-08-20) ++++++++++++++++++ From 7127bcda974303a1df19fbff22cccfe22fb8aac7 Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Fri, 16 Sep 2011 00:32:18 +0200 Subject: [PATCH 049/255] add basic unit tests for requests.api --- tests/unit/test_requests_api.py | 71 +++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100755 tests/unit/test_requests_api.py diff --git a/tests/unit/test_requests_api.py b/tests/unit/test_requests_api.py new file mode 100755 index 00000000..98591c3b --- /dev/null +++ b/tests/unit/test_requests_api.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import unittest +import mock +import sys +import os +sys.path.append(os.getcwd()) + +try: + import omnijson as json +except ImportError: + import json + +import requests +from requests.models import Response + +class RequestsAPIUnitTests(unittest.TestCase): + """Requests API unit test cases.""" + + def setUp(self): + pass + + + def tearDown(self): + """Teardown.""" + pass + + + @mock.patch('requests.api.request') + def test_http_get(self, mock_request): + mock_request.return_value = Response() + requests.get('http://google.com') + mock_request.assert_called_once_with('get', 'http://google.com', + allow_redirects= True) + + @mock.patch('requests.api.request') + def test_http_head(self, mock_request): + mock_request.return_value = Response() + requests.head('http://google.com') + mock_request.assert_called_once_with('head', 'http://google.com', + allow_redirects= True) + + @mock.patch('requests.api.request') + def test_http_post(self, mock_request): + mock_request.return_value = Response() + requests.post('http://google.com', {}) + mock_request.assert_called_once_with('post', 'http://google.com', + data= {}) + + @mock.patch('requests.api.request') + def test_http_put(self, mock_request): + mock_request.return_value = Response() + requests.put('http://google.com', {}) + mock_request.assert_called_once_with('put', 'http://google.com', + data= {}) + + @mock.patch('requests.api.request') + def test_http_patch(self, mock_request): + mock_request.return_value = Response() + requests.patch('http://google.com', {}) + mock_request.assert_called_once_with('patch', 'http://google.com', + data= {}) + + @mock.patch('requests.api.request') + def test_http_delete(self, mock_request): + mock_request.return_value = Response() + requests.delete('http://google.com') + mock_request.assert_called_once_with('delete', 'http://google.com') + +if __name__ == '__main__': + unittest.main() From f996313aabec649cab6b7a2b90763beab70aee3b Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Fri, 16 Sep 2011 00:33:03 +0200 Subject: [PATCH 050/255] move original tests to tests/integration --- test_requests.py => tests/integration/test_requests.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test_requests.py => tests/integration/test_requests.py (100%) diff --git a/test_requests.py b/tests/integration/test_requests.py similarity index 100% rename from test_requests.py rename to tests/integration/test_requests.py From 14081bed46399cac578182d48464b954e0fcdc17 Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Sat, 17 Sep 2011 17:25:27 +0200 Subject: [PATCH 051/255] add unit test for api.request --- tests/unit/test_requests_api.py | 39 +++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/unit/test_requests_api.py b/tests/unit/test_requests_api.py index 98591c3b..7835fdaa 100755 --- a/tests/unit/test_requests_api.py +++ b/tests/unit/test_requests_api.py @@ -26,6 +26,45 @@ class RequestsAPIUnitTests(unittest.TestCase): pass + @mock.patch('requests.api.dispatch_hook') + @mock.patch('requests.api.Request') + @mock.patch('requests.api.cookiejar_from_dict') + def test_request(self, mock_cjar, mock_request, mock_hook): + args = dict( + method = None, + url = None, + data = None, + params = None, + headers = None, + cookiejar = None, + files = None, + auth = None, + timeout = 1, + allow_redirects = None, + proxies = None, + ) + hooks = {'args': args, 'pre_request': mock_request, + 'post_request': mock_request, 'response': 'response'} + sideeffect = lambda x,y,z: hooks[x] + mock_cjar.return_value = None + mock_request.send = mock.Mock(return_value={}) + mock_request.response = "response" + mock_hook.side_effect = sideeffect + + r = requests.request('get','http://google.com') + + + mock_cjar.assert_called_once_with({}) + mock_hook.assert_called__with('args', None, args) + mock_request.assert_called_once_with(**args) + mock_hook.assert_called__with('pre_request', None, mock_request) + mock_request.send.assert_called_once_with() + mock_hook.assert_called__with('post_request', None, mock_request) + mock_hook.assert_called__with('response', None, mock_request) + self.assertEqual(r, "response") + + + @mock.patch('requests.api.request') def test_http_get(self, mock_request): mock_request.return_value = Response() From cb2e87210c0af4217aa9a1159a08c5634cd85ab2 Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Sat, 17 Sep 2011 17:50:19 +0200 Subject: [PATCH 052/255] add test with kwargs for api.get --- tests/unit/test_requests_api.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/unit/test_requests_api.py b/tests/unit/test_requests_api.py index 7835fdaa..62f0b6cb 100755 --- a/tests/unit/test_requests_api.py +++ b/tests/unit/test_requests_api.py @@ -72,6 +72,22 @@ class RequestsAPIUnitTests(unittest.TestCase): mock_request.assert_called_once_with('get', 'http://google.com', allow_redirects= True) + @mock.patch('requests.api.request') + def test_http_get_with_kwargs(self, mock_request): + mock_request.return_value = Response() + requests.get('http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + mock_request.assert_called_once_with('get', 'http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + @mock.patch('requests.api.request') def test_http_head(self, mock_request): mock_request.return_value = Response() From 5c5c13a28f48770d826cdebfb7bd62dffd15c117 Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Sat, 17 Sep 2011 17:52:07 +0200 Subject: [PATCH 053/255] add test with kwargs for api.head --- tests/unit/test_requests_api.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/unit/test_requests_api.py b/tests/unit/test_requests_api.py index 62f0b6cb..bee0cf28 100755 --- a/tests/unit/test_requests_api.py +++ b/tests/unit/test_requests_api.py @@ -96,6 +96,21 @@ class RequestsAPIUnitTests(unittest.TestCase): allow_redirects= True) @mock.patch('requests.api.request') + def test_http_head_with_kwargs(self, mock_request): + mock_request.return_value = Response() + requests.head('http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + mock_request.assert_called_once_with('head', 'http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + def test_http_post(self, mock_request): mock_request.return_value = Response() requests.post('http://google.com', {}) From 29f7a5bcfa0c4b95df66218c78ac8d219af48563 Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Sat, 17 Sep 2011 17:53:01 +0200 Subject: [PATCH 054/255] add test with kwargs for api.post --- tests/unit/test_requests_api.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/unit/test_requests_api.py b/tests/unit/test_requests_api.py index bee0cf28..e9868cd8 100755 --- a/tests/unit/test_requests_api.py +++ b/tests/unit/test_requests_api.py @@ -111,6 +111,7 @@ class RequestsAPIUnitTests(unittest.TestCase): allow_redirects=False, proxies="proxies", hooks="hooks") + @mock.patch('requests.api.request') def test_http_post(self, mock_request): mock_request.return_value = Response() requests.post('http://google.com', {}) @@ -118,6 +119,22 @@ class RequestsAPIUnitTests(unittest.TestCase): data= {}) @mock.patch('requests.api.request') + def test_http_post_with_kwargs(self, mock_request): + mock_request.return_value = Response() + requests.post('http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + mock_request.assert_called_once_with('post', 'http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + + def test_http_put(self, mock_request): mock_request.return_value = Response() requests.put('http://google.com', {}) From 42c45df84d64b04da4702a94d7fa42d01501b218 Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Sat, 17 Sep 2011 17:53:27 +0200 Subject: [PATCH 055/255] add test with kwargs for api.put --- tests/unit/test_requests_api.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/unit/test_requests_api.py b/tests/unit/test_requests_api.py index e9868cd8..8c7725e9 100755 --- a/tests/unit/test_requests_api.py +++ b/tests/unit/test_requests_api.py @@ -135,6 +135,7 @@ class RequestsAPIUnitTests(unittest.TestCase): proxies="proxies", hooks="hooks") + @mock.patch('requests.api.request') def test_http_put(self, mock_request): mock_request.return_value = Response() requests.put('http://google.com', {}) @@ -142,6 +143,22 @@ class RequestsAPIUnitTests(unittest.TestCase): data= {}) @mock.patch('requests.api.request') + def test_http_put_with_kwargs(self, mock_request): + mock_request.return_value = Response() + requests.put('http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + mock_request.assert_called_once_with('put', 'http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + + def test_http_patch(self, mock_request): mock_request.return_value = Response() requests.patch('http://google.com', {}) From 1d1927bb2b957658212ef89eb3cefdcc0a942033 Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Sat, 17 Sep 2011 17:53:49 +0200 Subject: [PATCH 056/255] add test with kwargs for api.patch --- tests/unit/test_requests_api.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/unit/test_requests_api.py b/tests/unit/test_requests_api.py index 8c7725e9..aef2a35c 100755 --- a/tests/unit/test_requests_api.py +++ b/tests/unit/test_requests_api.py @@ -159,6 +159,7 @@ class RequestsAPIUnitTests(unittest.TestCase): proxies="proxies", hooks="hooks") + @mock.patch('requests.api.request') def test_http_patch(self, mock_request): mock_request.return_value = Response() requests.patch('http://google.com', {}) @@ -166,6 +167,21 @@ class RequestsAPIUnitTests(unittest.TestCase): data= {}) @mock.patch('requests.api.request') + def test_http_patch_with_kwargs(self, mock_request): + mock_request.return_value = Response() + requests.patch('http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + mock_request.assert_called_once_with('patch', 'http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + def test_http_delete(self, mock_request): mock_request.return_value = Response() requests.delete('http://google.com') From 1481e9ae6a3ef8f507b944840c98aea0f3f23920 Mon Sep 17 00:00:00 2001 From: Daniel Schauenberg Date: Sat, 17 Sep 2011 17:54:18 +0200 Subject: [PATCH 057/255] add test with kwargs for api.delete --- tests/unit/test_requests_api.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/unit/test_requests_api.py b/tests/unit/test_requests_api.py index aef2a35c..ae63ab85 100755 --- a/tests/unit/test_requests_api.py +++ b/tests/unit/test_requests_api.py @@ -182,10 +182,28 @@ class RequestsAPIUnitTests(unittest.TestCase): allow_redirects=False, proxies="proxies", hooks="hooks") + @mock.patch('requests.api.request') def test_http_delete(self, mock_request): mock_request.return_value = Response() requests.delete('http://google.com') mock_request.assert_called_once_with('delete', 'http://google.com') + @mock.patch('requests.api.request') + def test_http_delete_with_kwargs(self, mock_request): + mock_request.return_value = Response() + requests.delete('http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + mock_request.assert_called_once_with('delete', 'http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + + if __name__ == '__main__': unittest.main() From 784b6715591acbec35c6a5b20d57cec9c5b448e8 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 17 Sep 2011 16:28:51 -0400 Subject: [PATCH 058/255] mock reqs for tests --- reqs.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 reqs.txt diff --git a/reqs.txt b/reqs.txt new file mode 100644 index 00000000..23ee03b2 --- /dev/null +++ b/reqs.txt @@ -0,0 +1 @@ +mock \ No newline at end of file From 84434fddee4b1423240e66de5b5c7c0ecaf6615d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 17 Sep 2011 20:30:08 -0400 Subject: [PATCH 059/255] import urllib3 --- requests/packages/urllib3/__init__.py | 22 + requests/packages/urllib3/connectionpool.py | 544 ++++++++++++++++++ requests/packages/urllib3/contrib/__init__.py | 0 requests/packages/urllib3/contrib/ntlmpool.py | 111 ++++ requests/packages/urllib3/filepost.py | 50 ++ 5 files changed, 727 insertions(+) create mode 100644 requests/packages/urllib3/__init__.py create mode 100644 requests/packages/urllib3/connectionpool.py create mode 100644 requests/packages/urllib3/contrib/__init__.py create mode 100644 requests/packages/urllib3/contrib/ntlmpool.py create mode 100644 requests/packages/urllib3/filepost.py diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py new file mode 100644 index 00000000..c7ade88e --- /dev/null +++ b/requests/packages/urllib3/__init__.py @@ -0,0 +1,22 @@ +""" +urllib3 - Thread-safe connection pooling and re-using. +""" + +from connectionpool import ( + connection_from_url, + get_host, + HTTPConnectionPool, + HTTPSConnectionPool, + make_headers) +# Possible exceptions +from connectionpool import ( + HTTPError, + MaxRetryError, + SSLError, + TimeoutError) +from filepost import encode_multipart_formdata + + +__author__ = "Andrey Petrov (andrey.petrov@shazow.net)" +__license__ = "MIT" +__version__ = "$Rev$" diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py new file mode 100644 index 00000000..552ccd98 --- /dev/null +++ b/requests/packages/urllib3/connectionpool.py @@ -0,0 +1,544 @@ +import gzip +import zlib +import logging +import socket + + +from urllib import urlencode +from httplib import HTTPConnection, HTTPSConnection, HTTPException +from Queue import Queue, Empty, Full +from select import select +from socket import error as SocketError, timeout as SocketTimeout + + +try: + import ssl + BaseSSLError = ssl.SSLError +except ImportError, e: + ssl = None + BaseSSLError = None + +try: + from cStringIO import StringIO +except ImportError, e: + from StringIO import StringIO + + +from filepost import encode_multipart_formdata + + +log = logging.getLogger(__name__) + + +## Exceptions + +class HTTPError(Exception): + "Base exception used by this module." + pass + + +class SSLError(Exception): + "Raised when SSL certificate fails in an HTTPS connection." + pass + + +class MaxRetryError(HTTPError): + "Raised when the maximum number of retries is exceeded." + pass + + +class TimeoutError(HTTPError): + "Raised when a socket timeout occurs." + pass + + +class HostChangedError(HTTPError): + "Raised when an existing pool gets a request for a foreign host." + pass + + +## Response objects + +class HTTPResponse(object): + """ + HTTP Response container. + + Similar to httplib's HTTPResponse but the data is pre-loaded. + """ + + def __init__(self, data='', headers=None, status=0, version=0, reason=None, + strict=0): + self.data = data + self.headers = headers or {} + self.status = status + self.version = version + self.reason = reason + self.strict = strict + + @staticmethod + def from_httplib(r): + """ + Given an httplib.HTTPResponse instance, return a corresponding + urllib3.HTTPResponse object. + + NOTE: This method will perform r.read() which will have side effects + on the original http.HTTPResponse object. + """ + tmp_data = r.read() + try: + if r.getheader('content-encoding') == 'gzip': + log.debug("Received response with content-encoding: gzip, " + "decompressing with gzip.") + + gzipper = gzip.GzipFile(fileobj=StringIO(tmp_data)) + data = gzipper.read() + elif r.getheader('content-encoding') == 'deflate': + log.debug("Received response with content-encoding: deflate, " + "decompressing with zlib.") + try: + data = zlib.decompress(tmp_data) + except zlib.error, e: + data = zlib.decompress(tmp_data, -zlib.MAX_WBITS) + else: + data = tmp_data + + except IOError: + raise HTTPError("Received response with content-encoding: %s, " + "but failed to decompress it." % + (r.getheader('content-encoding'))) + + return HTTPResponse(data=data, + headers=dict(r.getheaders()), + status=r.status, + version=r.version, + reason=r.reason, + strict=r.strict) + + # Backwards-compatibility methods for httplib.HTTPResponse + def getheaders(self): + return self.headers + + def getheader(self, name, default=None): + return self.headers.get(name, default) + + +## Connection objects + +class VerifiedHTTPSConnection(HTTPSConnection): + """ + Based on httplib.HTTPSConnection but wraps the socket with + SSL certification. + """ + + def set_cert(self, key_file=None, cert_file=None, cert_reqs='CERT_NONE', + ca_certs=None): + ssl_req_scheme = { + 'CERT_NONE': ssl.CERT_NONE, + 'CERT_OPTIONAL': ssl.CERT_OPTIONAL, + 'CERT_REQUIRED': ssl.CERT_REQUIRED + } + + self.key_file = key_file + self.cert_file = cert_file + self.cert_reqs = ssl_req_scheme.get(cert_reqs) or ssl.CERT_NONE + self.ca_certs = ca_certs + + def connect(self): + # Add certificate verification + sock = socket.create_connection((self.host, self.port), self.timeout) + + # Wrap socket using verification with the root certs in + # trusted_root_certs + self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, + cert_reqs=self.cert_reqs, + ca_certs=self.ca_certs) + + +## Pool objects + +class HTTPConnectionPool(object): + """ + Thread-safe connection pool for one host. + + host + Host used for this HTTP Connection (e.g. "localhost"), passed into + httplib.HTTPConnection() + + port + Port used for this HTTP Connection (None is equivalent to 80), passed + into httplib.HTTPConnection() + + strict + Causes BadStatusLine to be raised if the status line can't be parsed + as a valid HTTP/1.0 or 1.1 status line, passed into + httplib.HTTPConnection() + + timeout + Socket timeout for each individual connection, can be a float. None + disables timeout. + + maxsize + Number of connections to save that can be reused. More than 1 is useful + in multithreaded situations. If ``block`` is set to false, more + connections will be created but they will not be saved once they've + been used. + + block + If set to True, no more than ``maxsize`` connections will be used at + a time. When no free connections are available, the call will block + until a connection has been released. This is a useful side effect for + particular multithreaded situations where one does not want to use more + than maxsize connections per host to prevent flooding. + + headers + Headers to include with all requests, unless other headers are given + explicitly. + """ + + scheme = 'http' + + def __init__(self, host, port=None, strict=False, timeout=None, maxsize=1, + block=False, headers=None): + self.host = host + self.port = port + self.strict = strict + self.timeout = timeout + self.pool = Queue(maxsize) + self.block = block + self.headers = headers or {} + + # Fill the queue up so that doing get() on it will block properly + [self.pool.put(None) for i in xrange(maxsize)] + + self.num_connections = 0 + self.num_requests = 0 + + def _new_conn(self): + """ + Return a fresh HTTPConnection. + """ + self.num_connections += 1 + log.info("Starting new HTTP connection (%d): %s" % + (self.num_connections, self.host)) + return HTTPConnection(host=self.host, port=self.port) + + def _get_conn(self, timeout=None): + """ + Get a connection. Will return a pooled connection if one is available. + Otherwise, a fresh connection is returned. + """ + conn = None + try: + conn = self.pool.get(block=self.block, timeout=timeout) + + # If this is a persistent connection, check if it got disconnected + if conn and conn.sock and select([conn.sock], [], [], 0.0)[0]: + # Either data is buffered (bad), or the connection is dropped. + log.warning("Connection pool detected dropped " + "connection, resetting: %s" % self.host) + conn.close() + + except Empty, e: + pass # Oh well, we'll create a new connection then + + return conn or self._new_conn() + + def _put_conn(self, conn): + """ + Put a connection back into the pool. + If the pool is already full, the connection is discarded because we + exceeded maxsize. If connections are discarded frequently, then maxsize + should be increased. + """ + try: + self.pool.put(conn, block=False) + except Full, e: + # This should never happen if self.block == True + log.warning("HttpConnectionPool is full, discarding connection: %s" + % self.host) + + def is_same_host(self, url): + return (url.startswith('/') or + get_host(url) == (self.scheme, self.host, self.port)) + + def urlopen(self, method, url, body=None, headers=None, retries=3, + redirect=True, assert_same_host=True): + """ + Get a connection from the pool and perform an HTTP request. + + method + HTTP request method (such as GET, POST, PUT, etc.) + + body + Data to send in the request body (useful for creating + POST requests, see HTTPConnectionPool.post_url for + more convenience). + + headers + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. If None, pool headers are used. If provided, + these headers completely replace any pool-specific headers. + + retries + Number of retries to allow before raising + a MaxRetryError exception. + + redirect + Automatically handle redirects (status codes 301, 302, 303, 307), + each redirect counts as a retry. + + assert_same_host + If True, will make sure that the host of the pool requests is + consistent else will raise HostChangedError. When False, you can + use the pool on an HTTP proxy and request foreign hosts. + """ + if headers == None: + headers = self.headers + + if retries < 0: + raise MaxRetryError("Max retries exceeded for url: %s" % url) + + # Check host + if assert_same_host and not self.is_same_host(url): + host = "%s://%s" % (self.scheme, self.host) + if self.port: + host = "%s:%d" % (host, self.port) + + raise HostChangedError("Connection pool with host '%s' tried to " + "open a foreign host: %s" % (host, url)) + + try: + # Request a connection from the queue + conn = self._get_conn() + + # Make the request + self.num_requests += 1 + conn.request(method, url, body=body, headers=headers) + conn.sock.settimeout(self.timeout) + httplib_response = conn.getresponse() + log.debug("\"%s %s %s\" %s %s" % + (method, url, conn._http_vsn_str, + httplib_response.status, httplib_response.length)) + + # from_httplib will perform httplib_response.read() which will have + # the side effect of letting us use this connection for another + # request. + response = HTTPResponse.from_httplib(httplib_response) + + # Put the connection back to be reused + self._put_conn(conn) + + except (SocketTimeout, Empty), e: + # Timed out either by socket or queue + raise TimeoutError("Request timed out after %f seconds" % + self.timeout) + + except (BaseSSLError), e: + # SSL certificate error + raise SSLError(e) + + except (HTTPException, SocketError), e: + log.warn("Retrying (%d attempts remain) after connection " + "broken by '%r': %s" % (retries, e, url)) + return self.urlopen(method, url, body, headers, retries - 1, + redirect, assert_same_host) # Try again + + # Handle redirection + if (redirect and + response.status in [301, 302, 303, 307] and + 'location' in response.headers): # Redirect, retry + log.info("Redirecting %s -> %s" % + (url, response.headers.get('location'))) + return self.urlopen(method, response.headers.get('location'), body, + headers, retries - 1, redirect, + assert_same_host) + + return response + + def get_url(self, url, fields=None, headers=None, retries=3, + redirect=True): + """ + Wrapper for performing GET with urlopen (see urlopen for more details). + + Supports an optional ``fields`` dictionary parameter key/value strings. + If provided, they will be added to the url. + """ + if fields: + url += '?' + urlencode(fields) + return self.urlopen('GET', url, headers=headers, retries=retries, + redirect=redirect) + + def post_url(self, url, fields=None, headers=None, retries=3, + redirect=True, encode_multipart=True): + """ + Wrapper for performing POST with urlopen (see urlopen + for more details). + + Supports an optional ``fields`` parameter of key/value strings AND + key/filetuple. A filetuple is a (filename, data) tuple. For example: + + fields = { + 'foo': 'bar', + 'foofile': ('foofile.txt', 'contents of foofile'), + } + + If encode_multipart=True (default), then + ``urllib3.filepost.encode_multipart_formdata`` is used to encode the + payload with the appropriate content type. Otherwise + ``urllib.urlencode`` is used with 'application/x-www-form-urlencoded' + content type. + + Multipart encoding must be used when posting files, and it's reasonably + safe to use it other times too. It may break request signing, such as + OAuth. + + NOTE: If ``headers`` are supplied, the 'Content-Type' value will be + overwritten because it depends on the dynamic random boundary string + which is used to compose the body of the request. + """ + if encode_multipart: + body, content_type = encode_multipart_formdata(fields or {}) + else: + body, content_type = ( + urlencode(fields or {}), + 'application/x-www-form-urlencoded') + + headers = headers or {} + headers.update({'Content-Type': content_type}) + + return self.urlopen('POST', url, body, headers=headers, + retries=retries, redirect=redirect) + + +class HTTPSConnectionPool(HTTPConnectionPool): + """ + Same as HTTPConnectionPool, but HTTPS. + """ + + scheme = 'https' + + def __init__(self, host, port=None, strict=False, timeout=None, maxsize=1, + block=False, headers=None, key_file=None, + cert_file=None, cert_reqs='CERT_NONE', ca_certs=None): + self.host = host + self.port = port + self.strict = strict + self.timeout = timeout + self.pool = Queue(maxsize) + self.block = block + self.headers = headers or {} + + self.key_file = key_file + self.cert_file = cert_file + self.cert_reqs = cert_reqs + self.ca_certs = ca_certs + + # Fill the queue up so that doing get() on it will block properly + [self.pool.put(None) for i in xrange(maxsize)] + + self.num_connections = 0 + self.num_requests = 0 + + def _new_conn(self): + """ + Return a fresh HTTPSConnection. + """ + self.num_connections += 1 + log.info("Starting new HTTPS connection (%d): %s" + % (self.num_connections, self.host)) + + if not ssl: + return HTTPSConnection(host=self.host, port=self.port) + + connection = VerifiedHTTPSConnection(host=self.host, port=self.port) + connection.set_cert(key_file=self.key_file, cert_file=self.cert_file, + cert_reqs=self.cert_reqs, ca_certs=self.ca_certs) + return connection + + +## Helpers + +def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, + basic_auth=None): + """ + Shortcuts for generating request headers. + + keep_alive + If true, adds 'connection: keep-alive' header. + + accept_encoding + Can be a boolean, list, or string. + True translates to 'gzip,deflate'. + List will get joined by comma. + String will be used as provided. + + user_agent + String representing the user-agent you want, such as + "python-urllib3/0.6" + + basic_auth + Colon-separated username:password string for 'authorization: basic ...' + auth header. + """ + headers = {} + if accept_encoding: + if isinstance(accept_encoding, str): + pass + elif isinstance(accept_encoding, list): + accept_encoding = ','.join(accept_encoding) + else: + accept_encoding = 'gzip,deflate' + headers['accept-encoding'] = accept_encoding + + if user_agent: + headers['user-agent'] = user_agent + + if keep_alive: + headers['connection'] = 'keep-alive' + + if basic_auth: + headers['authorization'] = 'Basic ' + \ + basic_auth.encode('base64').strip() + + return headers + + +def get_host(url): + """ + Given a url, return its scheme, host and port (None if it's not there). + + For example: + >>> get_host('http://google.com/mail/') + http, google.com, None + >>> get_host('google.com:80') + http, google.com, 80 + """ + # This code is actually similar to urlparse.urlsplit, but much + # simplified for our needs. + port = None + scheme = 'http' + if '//' in url: + scheme, url = url.split('://', 1) + if '/' in url: + url, path = url.split('/', 1) + if ':' in url: + url, port = url.split(':', 1) + port = int(port) + return scheme, url, port + + +def connection_from_url(url, **kw): + """ + Given a url, return an HTTP(S)ConnectionPool instance of its host. + + This is a shortcut for not having to determine the host of the url + before creating an HTTP(S)ConnectionPool instance. + + Passes on whatever kw arguments to the constructor of + HTTP(S)ConnectionPool. (e.g. timeout, maxsize, block) + """ + scheme, host, port = get_host(url) + if scheme == 'https': + return HTTPSConnectionPool(host, port=port, **kw) + else: + return HTTPConnectionPool(host, port=port, **kw) diff --git a/requests/packages/urllib3/contrib/__init__.py b/requests/packages/urllib3/contrib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/requests/packages/urllib3/contrib/ntlmpool.py b/requests/packages/urllib3/contrib/ntlmpool.py new file mode 100644 index 00000000..a4642fac --- /dev/null +++ b/requests/packages/urllib3/contrib/ntlmpool.py @@ -0,0 +1,111 @@ +""" +NTLM authenticating pool, contributed by erikcederstran + +Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10 +""" + +import httplib +from logging import getLogger +from ntlm import ntlm + +from urllib3 import HTTPSConnectionPool + + +log = getLogger(__name__) + + +class NTLMConnectionPool(HTTPSConnectionPool): + """ + Implements an NTLM authentication version of an urllib3 connection pool + """ + + scheme = 'https' + + def __init__(self, user, pw, authurl, *args, **kwargs): + """ + authurl is a random URL on the server that is protected by NTLM. + user is the Windows user, probably in the DOMAIN\username format. + pw is the password for the user. + """ + super(NTLMConnectionPool, self).__init__(*args, **kwargs) + self.authurl = authurl + self.rawuser = user + user_parts = user.split('\\', 1) + self.domain = user_parts[0].upper() + self.user = user_parts[1] + self.pw = pw + + def _new_conn(self): + # Performs the NTLM handshake that secures the connection. The socket + # must be kept open while requests are performed. + self.num_connections += 1 + log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s' % + (self.num_connections, self.host, self.authurl)) + + headers = {} + headers['Connection'] = 'Keep-Alive' + req_header = 'Authorization' + resp_header = 'www-authenticate' + + conn = httplib.HTTPSConnection(host=self.host, port=self.port) + + # Send negotiation message + headers[req_header] = ( + 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)) + log.debug('Request headers: %s' % headers) + conn.request('GET', self.authurl, None, headers) + res = conn.getresponse() + reshdr = dict(res.getheaders()) + log.debug('Response status: %s %s' % (res.status, res.reason)) + log.debug('Response headers: %s' % reshdr) + log.debug('Response data: %s [...]' % res.read(100)) + + # Remove the reference to the socket, so that it can not be closed by + # the response object (we want to keep the socket open) + res.fp = None + + # Server should respond with a challenge message + auth_header_values = reshdr[resp_header].split(', ') + auth_header_value = None + for s in auth_header_values: + if s[:5] == 'NTLM ': + auth_header_value = s[5:] + if auth_header_value is None: + raise Exception('Unexpected %s response header: %s' % + (resp_header, reshdr[resp_header])) + + # Send authentication message + ServerChallenge, NegotiateFlags = \ + ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value) + auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, + self.user, + self.domain, + self.pw, + NegotiateFlags) + headers[req_header] = 'NTLM %s' % auth_msg + log.debug('Request headers: %s' % headers) + conn.request('GET', self.authurl, None, headers) + res = conn.getresponse() + log.debug('Response status: %s %s' % (res.status, res.reason)) + log.debug('Response headers: %s' % dict(res.getheaders())) + log.debug('Response data: %s [...]' % res.read()[:100]) + if res.status != 200: + if res.status == 401: + raise Exception('Server rejected request: wrong ' + 'username or password') + raise Exception('Wrong server response: %s %s' % + (res.status, res.reason)) + + res.fp = None + log.debug('Connection established') + return conn + + def urlopen(self, method, url, body=None, headers=None, retries=3, + redirect=True, assert_same_host=True): + if headers is None: + headers = {} + headers['Connection'] = 'Keep-Alive' + return super(NTLMConnectionPool, self).urlopen(method, url, body, + headers, retries, + redirect, + assert_same_host) diff --git a/requests/packages/urllib3/filepost.py b/requests/packages/urllib3/filepost.py new file mode 100644 index 00000000..181822ba --- /dev/null +++ b/requests/packages/urllib3/filepost.py @@ -0,0 +1,50 @@ +import codecs +import mimetools +import mimetypes +try: + from cStringIO import StringIO +except: + from StringIO import StringIO + + +writer = codecs.lookup('utf-8')[3] + + +def get_content_type(filename): + return mimetypes.guess_type(filename)[0] or 'application/octet-stream' + + +def encode_multipart_formdata(fields): + body = StringIO() + BOUNDARY = mimetools.choose_boundary() + + for fieldname, value in fields.iteritems(): + body.write('--%s\r\n' % (BOUNDARY)) + + if isinstance(value, tuple): + filename, data = value + writer(body).write('Content-Disposition: form-data; name="%s"; ' + 'filename="%s"\r\n' % (fieldname, filename)) + body.write('Content-Type: %s\r\n\r\n' % + (get_content_type(filename))) + else: + data = value + writer(body).write('Content-Disposition: form-data; name="%s"\r\n' + % (fieldname)) + body.write('Content-Type: text/plain\r\n\r\n') + + if isinstance(data, int): + data = str(data) # Backwards compatibility + + if isinstance(data, unicode): + writer(body).write(data) + else: + body.write(data) + + body.write('\r\n') + + body.write('--%s--\r\n' % (BOUNDARY)) + + content_type = 'multipart/form-data; boundary=%s' % BOUNDARY + + return body.getvalue(), content_type From 3dd2a0443d44a8e30d77ddf55c3fba39062dc6f4 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 17 Sep 2011 20:31:56 -0400 Subject: [PATCH 060/255] remove notice file --- NOTICE | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 NOTICE diff --git a/NOTICE b/NOTICE deleted file mode 100644 index 08ba2bc6..00000000 --- a/NOTICE +++ /dev/null @@ -1,12 +0,0 @@ -Request includes some vendorized python libraries to ease installation. - -Poster License -============== - -Copyright (c) 2010 Chris AtLee - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. From 608c61dcd97961fee57d2f3de710229539bb16c4 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 17 Sep 2011 20:32:13 -0400 Subject: [PATCH 061/255] remove poster! \o/ --- requests/packages/poster/__init__.py | 34 -- requests/packages/poster/encode.py | 414 ---------------------- requests/packages/poster/streaminghttp.py | 199 ----------- 3 files changed, 647 deletions(-) delete mode 100644 requests/packages/poster/__init__.py delete mode 100644 requests/packages/poster/encode.py delete mode 100644 requests/packages/poster/streaminghttp.py diff --git a/requests/packages/poster/__init__.py b/requests/packages/poster/__init__.py deleted file mode 100644 index 6e216fce..00000000 --- a/requests/packages/poster/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2010 Chris AtLee -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -"""poster module - -Support for streaming HTTP uploads, and multipart/form-data encoding - -```poster.version``` is a 3-tuple of integers representing the version number. -New releases of poster will always have a version number that compares greater -than an older version of poster. -New in version 0.6.""" - -from __future__ import absolute_import - -from . import streaminghttp -from . import encode - -version = (0, 8, 0) # Thanks JP! diff --git a/requests/packages/poster/encode.py b/requests/packages/poster/encode.py deleted file mode 100644 index cf2298d7..00000000 --- a/requests/packages/poster/encode.py +++ /dev/null @@ -1,414 +0,0 @@ -"""multipart/form-data encoding module - -This module provides functions that faciliate encoding name/value pairs -as multipart/form-data suitable for a HTTP POST or PUT request. - -multipart/form-data is the standard way to upload files over HTTP""" - -__all__ = ['gen_boundary', 'encode_and_quote', 'MultipartParam', - 'encode_string', 'encode_file_header', 'get_body_size', 'get_headers', - 'multipart_encode'] - -try: - import uuid - def gen_boundary(): - """Returns a random string to use as the boundary for a message""" - return uuid.uuid4().hex -except ImportError: - import random, sha - def gen_boundary(): - """Returns a random string to use as the boundary for a message""" - bits = random.getrandbits(160) - return sha.new(str(bits)).hexdigest() - -import urllib, re, os, mimetypes -try: - from email.header import Header -except ImportError: - # Python 2.4 - from email.Header import Header - -def encode_and_quote(data): - """If ``data`` is unicode, return urllib.quote_plus(data.encode("utf-8")) - otherwise return urllib.quote_plus(data)""" - if data is None: - return None - - if isinstance(data, unicode): - data = data.encode("utf-8") - return urllib.quote_plus(data) - -def _strify(s): - """If s is a unicode string, encode it to UTF-8 and return the results, - otherwise return str(s), or None if s is None""" - if s is None: - return None - if isinstance(s, unicode): - return s.encode("utf-8") - return str(s) - -class MultipartParam(object): - """Represents a single parameter in a multipart/form-data request - - ``name`` is the name of this parameter. - - If ``value`` is set, it must be a string or unicode object to use as the - data for this parameter. - - If ``filename`` is set, it is what to say that this parameter's filename - is. Note that this does not have to be the actual filename any local file. - - If ``filetype`` is set, it is used as the Content-Type for this parameter. - If unset it defaults to "text/plain; charset=utf8" - - If ``filesize`` is set, it specifies the length of the file ``fileobj`` - - If ``fileobj`` is set, it must be a file-like object that supports - .read(). - - Both ``value`` and ``fileobj`` must not be set, doing so will - raise a ValueError assertion. - - If ``fileobj`` is set, and ``filesize`` is not specified, then - the file's size will be determined first by stat'ing ``fileobj``'s - file descriptor, and if that fails, by seeking to the end of the file, - recording the current position as the size, and then by seeking back to the - beginning of the file. - - ``cb`` is a callable which will be called from iter_encode with (self, - current, total), representing the current parameter, current amount - transferred, and the total size. - """ - def __init__(self, name, value=None, filename=None, filetype=None, - filesize=None, fileobj=None, cb=None): - self.name = Header(name).encode() - self.value = _strify(value) - if filename is None: - self.filename = None - else: - if isinstance(filename, unicode): - # Encode with XML entities - self.filename = filename.encode("ascii", "xmlcharrefreplace") - else: - self.filename = str(filename) - self.filename = self.filename.encode("string_escape").\ - replace('"', '\\"') - self.filetype = _strify(filetype) - - self.filesize = filesize - self.fileobj = fileobj - self.cb = cb - - if self.value is not None and self.fileobj is not None: - raise ValueError("Only one of value or fileobj may be specified") - - if fileobj is not None and filesize is None: - # Try and determine the file size - try: - self.filesize = os.fstat(fileobj.fileno()).st_size - except (OSError, AttributeError): - try: - fileobj.seek(0, 2) - self.filesize = fileobj.tell() - fileobj.seek(0) - except: - raise ValueError("Could not determine filesize") - - def __cmp__(self, other): - attrs = ['name', 'value', 'filename', 'filetype', 'filesize', 'fileobj'] - myattrs = [getattr(self, a) for a in attrs] - oattrs = [getattr(other, a) for a in attrs] - return cmp(myattrs, oattrs) - - def reset(self): - if self.fileobj is not None: - self.fileobj.seek(0) - elif self.value is None: - raise ValueError("Don't know how to reset this parameter") - - @classmethod - def from_file(cls, paramname, filename): - """Returns a new MultipartParam object constructed from the local - file at ``filename``. - - ``filesize`` is determined by os.path.getsize(``filename``) - - ``filetype`` is determined by mimetypes.guess_type(``filename``)[0] - - ``filename`` is set to os.path.basename(``filename``) - """ - - return cls(paramname, filename=os.path.basename(filename), - filetype=mimetypes.guess_type(filename)[0], - filesize=os.path.getsize(filename), - fileobj=open(filename, "rb")) - - @classmethod - def from_params(cls, params): - """Returns a list of MultipartParam objects from a sequence of - name, value pairs, MultipartParam instances, - or from a mapping of names to values - - The values may be strings or file objects, or MultipartParam objects. - MultipartParam object names must match the given names in the - name,value pairs or mapping, if applicable.""" - if hasattr(params, 'items'): - params = params.items() - - retval = [] - for item in params: - if isinstance(item, cls): - retval.append(item) - continue - name, value = item - if isinstance(value, cls): - assert value.name == name - retval.append(value) - continue - if hasattr(value, 'read'): - # Looks like a file object - filename = getattr(value, 'name', None) - if filename is not None: - filetype = mimetypes.guess_type(filename)[0] - else: - filetype = None - - retval.append(cls(name=name, filename=filename, - filetype=filetype, fileobj=value)) - else: - retval.append(cls(name, value)) - return retval - - def encode_hdr(self, boundary): - """Returns the header of the encoding of this parameter""" - boundary = encode_and_quote(boundary) - - headers = ["--%s" % boundary] - - if self.filename: - disposition = 'form-data; name="%s"; filename="%s"' % (self.name, - self.filename) - else: - disposition = 'form-data; name="%s"' % self.name - - headers.append("Content-Disposition: %s" % disposition) - - if self.filetype: - filetype = self.filetype - else: - filetype = "text/plain; charset=utf-8" - - headers.append("Content-Type: %s" % filetype) - - headers.append("") - headers.append("") - - return "\r\n".join(headers) - - def encode(self, boundary): - """Returns the string encoding of this parameter""" - if self.value is None: - value = self.fileobj.read() - else: - value = self.value - - if re.search("^--%s$" % re.escape(boundary), value, re.M): - raise ValueError("boundary found in encoded string") - - return "%s%s\r\n" % (self.encode_hdr(boundary), value) - - def iter_encode(self, boundary, blocksize=4096): - """Yields the encoding of this parameter - If self.fileobj is set, then blocks of ``blocksize`` bytes are read and - yielded.""" - total = self.get_size(boundary) - current = 0 - if self.value is not None: - block = self.encode(boundary) - current += len(block) - yield block - if self.cb: - self.cb(self, current, total) - else: - block = self.encode_hdr(boundary) - current += len(block) - yield block - if self.cb: - self.cb(self, current, total) - last_block = "" - encoded_boundary = "--%s" % encode_and_quote(boundary) - boundary_exp = re.compile("^%s$" % re.escape(encoded_boundary), - re.M) - while True: - block = self.fileobj.read(blocksize) - if not block: - current += 2 - yield "\r\n" - if self.cb: - self.cb(self, current, total) - break - last_block += block - if boundary_exp.search(last_block): - raise ValueError("boundary found in file data") - last_block = last_block[-len(encoded_boundary)-2:] - current += len(block) - yield block - if self.cb: - self.cb(self, current, total) - - def get_size(self, boundary): - """Returns the size in bytes that this param will be when encoded - with the given boundary.""" - if self.filesize is not None: - valuesize = self.filesize - else: - valuesize = len(self.value) - - return len(self.encode_hdr(boundary)) + 2 + valuesize - -def encode_string(boundary, name, value): - """Returns ``name`` and ``value`` encoded as a multipart/form-data - variable. ``boundary`` is the boundary string used throughout - a single request to separate variables.""" - - return MultipartParam(name, value).encode(boundary) - -def encode_file_header(boundary, paramname, filesize, filename=None, - filetype=None): - """Returns the leading data for a multipart/form-data field that contains - file data. - - ``boundary`` is the boundary string used throughout a single request to - separate variables. - - ``paramname`` is the name of the variable in this request. - - ``filesize`` is the size of the file data. - - ``filename`` if specified is the filename to give to this field. This - field is only useful to the server for determining the original filename. - - ``filetype`` if specified is the MIME type of this file. - - The actual file data should be sent after this header has been sent. - """ - - return MultipartParam(paramname, filesize=filesize, filename=filename, - filetype=filetype).encode_hdr(boundary) - -def get_body_size(params, boundary): - """Returns the number of bytes that the multipart/form-data encoding - of ``params`` will be.""" - size = sum(p.get_size(boundary) for p in MultipartParam.from_params(params)) - return size + len(boundary) + 6 - -def get_headers(params, boundary): - """Returns a dictionary with Content-Type and Content-Length headers - for the multipart/form-data encoding of ``params``.""" - headers = {} - boundary = urllib.quote_plus(boundary) - headers['Content-Type'] = "multipart/form-data; boundary=%s" % boundary - headers['Content-Length'] = str(get_body_size(params, boundary)) - return headers - -class multipart_yielder: - def __init__(self, params, boundary, cb): - self.params = params - self.boundary = boundary - self.cb = cb - - self.i = 0 - self.p = None - self.param_iter = None - self.current = 0 - self.total = get_body_size(params, boundary) - - def __iter__(self): - return self - - def next(self): - """generator function to yield multipart/form-data representation - of parameters""" - if self.param_iter is not None: - try: - block = self.param_iter.next() - self.current += len(block) - if self.cb: - self.cb(self.p, self.current, self.total) - return block - except StopIteration: - self.p = None - self.param_iter = None - - if self.i is None: - raise StopIteration - elif self.i >= len(self.params): - self.param_iter = None - self.p = None - self.i = None - block = "--%s--\r\n" % self.boundary - self.current += len(block) - if self.cb: - self.cb(self.p, self.current, self.total) - return block - - self.p = self.params[self.i] - self.param_iter = self.p.iter_encode(self.boundary) - self.i += 1 - return self.next() - - def reset(self): - self.i = 0 - self.current = 0 - for param in self.params: - param.reset() - -def multipart_encode(params, boundary=None, cb=None): - """Encode ``params`` as multipart/form-data. - - ``params`` should be a sequence of (name, value) pairs or MultipartParam - objects, or a mapping of names to values. - Values are either strings parameter values, or file-like objects to use as - the parameter value. The file-like objects must support .read() and either - .fileno() or both .seek() and .tell(). - - If ``boundary`` is set, then it as used as the MIME boundary. Otherwise - a randomly generated boundary will be used. In either case, if the - boundary string appears in the parameter values a ValueError will be - raised. - - If ``cb`` is set, it should be a callback which will get called as blocks - of data are encoded. It will be called with (param, current, total), - indicating the current parameter being encoded, the current amount encoded, - and the total amount to encode. - - Returns a tuple of `datagen`, `headers`, where `datagen` is a - generator that will yield blocks of data that make up the encoded - parameters, and `headers` is a dictionary with the assoicated - Content-Type and Content-Length headers. - - Examples: - - >>> datagen, headers = multipart_encode( [("key", "value1"), ("key", "value2")] ) - >>> s = "".join(datagen) - >>> assert "value2" in s and "value1" in s - - >>> p = MultipartParam("key", "value2") - >>> datagen, headers = multipart_encode( [("key", "value1"), p] ) - >>> s = "".join(datagen) - >>> assert "value2" in s and "value1" in s - - >>> datagen, headers = multipart_encode( {"key": "value1"} ) - >>> s = "".join(datagen) - >>> assert "value2" not in s and "value1" in s - - """ - if boundary is None: - boundary = gen_boundary() - else: - boundary = urllib.quote_plus(boundary) - - headers = get_headers(params, boundary) - params = MultipartParam.from_params(params) - - return multipart_yielder(params, boundary, cb), headers diff --git a/requests/packages/poster/streaminghttp.py b/requests/packages/poster/streaminghttp.py deleted file mode 100644 index 1b591d4b..00000000 --- a/requests/packages/poster/streaminghttp.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Streaming HTTP uploads module. - -This module extends the standard httplib and urllib2 objects so that -iterable objects can be used in the body of HTTP requests. - -In most cases all one should have to do is call :func:`register_openers()` -to register the new streaming http handlers which will take priority over -the default handlers, and then you can use iterable objects in the body -of HTTP requests. - -**N.B.** You must specify a Content-Length header if using an iterable object -since there is no way to determine in advance the total size that will be -yielded, and there is no way to reset an interator. - -Example usage: - ->>> from StringIO import StringIO ->>> import urllib2, poster.streaminghttp - ->>> opener = poster.streaminghttp.register_openers() - ->>> s = "Test file data" ->>> f = StringIO(s) - ->>> req = urllib2.Request("http://localhost:5000", f, -... {'Content-Length': str(len(s))}) -""" - -import httplib, urllib2, socket -from httplib import NotConnected - -__all__ = ['StreamingHTTPConnection', 'StreamingHTTPRedirectHandler', - 'StreamingHTTPHandler', 'register_openers'] - -if hasattr(httplib, 'HTTPS'): - __all__.extend(['StreamingHTTPSHandler', 'StreamingHTTPSConnection']) - -class _StreamingHTTPMixin: - """Mixin class for HTTP and HTTPS connections that implements a streaming - send method.""" - def send(self, value): - """Send ``value`` to the server. - - ``value`` can be a string object, a file-like object that supports - a .read() method, or an iterable object that supports a .next() - method. - """ - # Based on python 2.6's httplib.HTTPConnection.send() - if self.sock is None: - if self.auto_open: - self.connect() - else: - raise NotConnected() - - # send the data to the server. if we get a broken pipe, then close - # the socket. we want to reconnect when somebody tries to send again. - # - # NOTE: we DO propagate the error, though, because we cannot simply - # ignore the error... the caller will know if they can retry. - if self.debuglevel > 0: - print "send:", repr(value) - try: - blocksize = 8192 - if hasattr(value, 'read') : - if hasattr(value, 'seek'): - value.seek(0) - if self.debuglevel > 0: - print "sendIng a read()able" - data = value.read(blocksize) - while data: - self.sock.sendall(data) - data = value.read(blocksize) - elif hasattr(value, 'next'): - if hasattr(value, 'reset'): - value.reset() - if self.debuglevel > 0: - print "sendIng an iterable" - for data in value: - self.sock.sendall(data) - else: - self.sock.sendall(value) - except socket.error, v: - if v[0] == 32: # Broken pipe - self.close() - raise - -class StreamingHTTPConnection(_StreamingHTTPMixin, httplib.HTTPConnection): - """Subclass of `httplib.HTTPConnection` that overrides the `send()` method - to support iterable body objects""" - -class StreamingHTTPRedirectHandler(urllib2.HTTPRedirectHandler): - """Subclass of `urllib2.HTTPRedirectHandler` that overrides the - `redirect_request` method to properly handle redirected POST requests - - This class is required because python 2.5's HTTPRedirectHandler does - not remove the Content-Type or Content-Length headers when requesting - the new resource, but the body of the original request is not preserved. - """ - - handler_order = urllib2.HTTPRedirectHandler.handler_order - 1 - - # From python2.6 urllib2's HTTPRedirectHandler - def redirect_request(self, req, fp, code, msg, headers, newurl): - """Return a Request or None in response to a redirect. - - This is called by the http_error_30x methods when a - redirection response is received. If a redirection should - take place, return a new Request to allow http_error_30x to - perform the redirect. Otherwise, raise HTTPError if no-one - else should try to handle this url. Return None if you can't - but another Handler might. - """ - m = req.get_method() - if (code in (301, 302, 303, 307) and m in ("GET", "HEAD") - or code in (301, 302, 303) and m == "POST"): - # Strictly (according to RFC 2616), 301 or 302 in response - # to a POST MUST NOT cause a redirection without confirmation - # from the user (of urllib2, in this case). In practice, - # essentially all clients do redirect in this case, so we - # do the same. - # be conciliant with URIs containing a space - newurl = newurl.replace(' ', '%20') - newheaders = dict((k, v) for k, v in req.headers.items() - if k.lower() not in ( - "content-length", "content-type") - ) - return urllib2.Request(newurl, - headers=newheaders, - origin_req_host=req.get_origin_req_host(), - unverifiable=True) - else: - raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp) - -class StreamingHTTPHandler(urllib2.HTTPHandler): - """Subclass of `urllib2.HTTPHandler` that uses - StreamingHTTPConnection as its http connection class.""" - - handler_order = urllib2.HTTPHandler.handler_order - 1 - - def http_open(self, req): - """Open a StreamingHTTPConnection for the given request""" - return self.do_open(StreamingHTTPConnection, req) - - def http_request(self, req): - """Handle a HTTP request. Make sure that Content-Length is specified - if we're using an interable value""" - # Make sure that if we're using an iterable object as the request - # body, that we've also specified Content-Length - if req.has_data(): - data = req.get_data() - if hasattr(data, 'read') or hasattr(data, 'next'): - if not req.has_header('Content-length'): - raise ValueError( - "No Content-Length specified for iterable body") - return urllib2.HTTPHandler.do_request_(self, req) - -if hasattr(httplib, 'HTTPS'): - class StreamingHTTPSConnection(_StreamingHTTPMixin, - httplib.HTTPSConnection): - """Subclass of `httplib.HTTSConnection` that overrides the `send()` - method to support iterable body objects""" - - class StreamingHTTPSHandler(urllib2.HTTPSHandler): - """Subclass of `urllib2.HTTPSHandler` that uses - StreamingHTTPSConnection as its http connection class.""" - - handler_order = urllib2.HTTPSHandler.handler_order - 1 - - def https_open(self, req): - return self.do_open(StreamingHTTPSConnection, req) - - def https_request(self, req): - # Make sure that if we're using an iterable object as the request - # body, that we've also specified Content-Length - if req.has_data(): - data = req.get_data() - if hasattr(data, 'read') or hasattr(data, 'next'): - if not req.has_header('Content-length'): - raise ValueError( - "No Content-Length specified for iterable body") - return urllib2.HTTPSHandler.do_request_(self, req) - - -def get_handlers(): - handlers = [StreamingHTTPHandler, StreamingHTTPRedirectHandler] - if hasattr(httplib, "HTTPS"): - handlers.append(StreamingHTTPSHandler) - return handlers - -def register_openers(): - """Register the streaming http handlers in the global urllib2 default - opener object. - - Returns the created OpenerDirector object.""" - opener = urllib2.build_opener(*get_handlers()) - - urllib2.install_opener(opener) - - return opener From e69a3a41f630fd580256f3b9b4004cbca1da6665 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 17 Sep 2011 20:32:21 -0400 Subject: [PATCH 062/255] import urllib3! --- requests/packages/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/packages/__init__.py b/requests/packages/__init__.py index ab2669e8..d62c4b71 100644 --- a/requests/packages/__init__.py +++ b/requests/packages/__init__.py @@ -1,3 +1,3 @@ from __future__ import absolute_import -from . import poster +from . import urllib3 From 0cb7376db0c725574939aa12367a654b2e10aaeb Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 17 Sep 2011 20:34:30 -0400 Subject: [PATCH 063/255] urllib3 in authors --- AUTHORS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/AUTHORS b/AUTHORS index af00b19e..14b7b858 100644 --- a/AUTHORS +++ b/AUTHORS @@ -7,6 +7,12 @@ Development Lead - Kenneth Reitz +Urllib3 +``````` + +- Andrey Petrov + + Patches and Suggestions ``````````````````````` From 3127a34298ec4a674916033b8d9678a1b57f828b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 17 Sep 2011 20:41:10 -0400 Subject: [PATCH 064/255] remove invalidmethod --- requests/exceptions.py | 5 +---- requests/models.py | 3 +-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/requests/exceptions.py b/requests/exceptions.py index c08c6148..f6585f46 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -12,15 +12,12 @@ class RequestException(Exception): class AuthenticationError(RequestException): """The authentication credentials provided were invalid.""" - + class Timeout(RequestException): """The request timed out.""" class URLRequired(RequestException): """A valid URL is required to make a request.""" -class InvalidMethod(RequestException): - """An inappropriate method was attempted.""" - class TooManyRedirects(RequestException): """Too many redirects.""" diff --git a/requests/models.py b/requests/models.py index 742acf2c..9b6b0a53 100644 --- a/requests/models.py +++ b/requests/models.py @@ -9,7 +9,6 @@ requests.models import urllib import urllib2 import socket -import codecs import zlib @@ -24,7 +23,7 @@ from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers from .utils import dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip from .status_codes import codes -from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects +from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, TooManyRedirects REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) From aa5e47723621d86e249884c083b2d51d87135f47 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 17 Sep 2011 22:02:06 -0400 Subject: [PATCH 065/255] basic get request works --- requests/models.py | 79 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 9 deletions(-) diff --git a/requests/models.py b/requests/models.py index 9b6b0a53..36bba15d 100644 --- a/requests/models.py +++ b/requests/models.py @@ -16,11 +16,15 @@ from urllib2 import HTTPError from urlparse import urlparse, urlunparse, urljoin from datetime import datetime + +from .packages import urllib3 +print dir(urllib3) + from .config import settings from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler from .structures import CaseInsensitiveDict -from .packages.poster.encode import multipart_encode -from .packages.poster.streaminghttp import register_openers, get_handlers +# from .packages.poster.encode import multipart_encode +# from .packages.poster.streaminghttp import register_openers, get_handlers from .utils import dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip from .status_codes import codes from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, TooManyRedirects @@ -182,16 +186,15 @@ class Request(object): def build(resp): response = Response() - response.status_code = getattr(resp, 'code', None) + response.status_code = getattr(resp, 'status', None) try: - response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None)) - response.fo = resp + response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) + # response.fo = resp - if self.cookiejar: - - response.cookies = dict_from_cookiejar(self.cookiejar) + # if self.cookiejar: + # response.cookies = dict_from_cookiejar(self.cookiejar) except AttributeError: pass @@ -216,7 +219,7 @@ class Request(object): (self.allow_redirects)) ): - r.fo.close() + # r.fo.close() if not len(history) < settings.max_redirects: raise TooManyRedirects() @@ -302,6 +305,64 @@ class Request(object): def send(self, anyway=False): + """Sends the shit.""" + + # Safety check. + self._checks() + + # Build the final URL. + url = self._build_url() + + # Setup Files. + # Setup form data. + # + # def urlopen(self, method, url, body=None, headers=None, retries=3, + # redirect=True, assert_same_host=True): + # req = _Request(url, data=self._enc_data, method=self.method) + + + if not self.sent or anyway: + + try: + + pool = urllib3.connection_from_url(url, timeout=self.timeout) + + r = pool.urlopen( + method=self.method, + url=url, + body=self.data, + headers=self.headers, + redirect=False, + assert_same_host=False + ) + + r.socket = pool._get_conn().sock + + # if self.cookiejar is not None: + # self.cookiejar.extract_cookies(resp, req) + + # except (urllib2.HTTPError, urllib2.URLError), why: + except Exception, why: + # if hasattr(why, 'reason'): + # if isinstance(why.reason, socket.timeout): + # why = Timeout(why) + + # self._build_response(why, is_error=True) + print 'FUCK' + print why + + else: + self._build_response(r) + self.response.ok = True + + + self.sent = self.response.ok + + return self.sent + + + + def old_send(self, anyway=False): """Sends the request. Returns True of successful, false if not. If there was an HTTPError during transmission, self.response.status_code will contain the HTTPError code. From 71d3490bcde6a6c595bc76a1244f059a7d7f4458 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 17 Sep 2011 22:40:30 -0400 Subject: [PATCH 066/255] optional blocking in urllib3 (for now) --- requests/packages/urllib3/connectionpool.py | 55 ++++++++++++--------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 552ccd98..c4db2378 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -76,7 +76,7 @@ class HTTPResponse(object): self.strict = strict @staticmethod - def from_httplib(r): + def from_httplib(r, block=True): """ Given an httplib.HTTPResponse instance, return a corresponding urllib3.HTTPResponse object. @@ -84,36 +84,43 @@ class HTTPResponse(object): NOTE: This method will perform r.read() which will have side effects on the original http.HTTPResponse object. """ - tmp_data = r.read() - try: - if r.getheader('content-encoding') == 'gzip': - log.debug("Received response with content-encoding: gzip, " - "decompressing with gzip.") - gzipper = gzip.GzipFile(fileobj=StringIO(tmp_data)) - data = gzipper.read() - elif r.getheader('content-encoding') == 'deflate': - log.debug("Received response with content-encoding: deflate, " - "decompressing with zlib.") - try: - data = zlib.decompress(tmp_data) - except zlib.error, e: - data = zlib.decompress(tmp_data, -zlib.MAX_WBITS) - else: - data = tmp_data + if block: + tmp_data = r.read() + try: + if r.getheader('content-encoding') == 'gzip': + log.debug("Received response with content-encoding: gzip, " + "decompressing with gzip.") - except IOError: - raise HTTPError("Received response with content-encoding: %s, " - "but failed to decompress it." % - (r.getheader('content-encoding'))) + gzipper = gzip.GzipFile(fileobj=StringIO(tmp_data)) + data = gzipper.read() + elif r.getheader('content-encoding') == 'deflate': + log.debug("Received response with content-encoding: deflate, " + "decompressing with zlib.") + try: + data = zlib.decompress(tmp_data) + except zlib.error, e: + data = zlib.decompress(tmp_data, -zlib.MAX_WBITS) + else: + data = tmp_data - return HTTPResponse(data=data, + except IOError: + raise HTTPError("Received response with content-encoding: %s, " + "but failed to decompress it." % + (r.getheader('content-encoding'))) + else: + data = None + + resp = HTTPResponse(data=data, headers=dict(r.getheaders()), status=r.status, version=r.version, reason=r.reason, strict=r.strict) + resp._raw = r + return resp + # Backwards-compatibility methods for httplib.HTTPResponse def getheaders(self): return self.headers @@ -262,7 +269,7 @@ class HTTPConnectionPool(object): get_host(url) == (self.scheme, self.host, self.port)) def urlopen(self, method, url, body=None, headers=None, retries=3, - redirect=True, assert_same_host=True): + redirect=True, assert_same_host=True, block=True): """ Get a connection from the pool and perform an HTTP request. @@ -323,7 +330,7 @@ class HTTPConnectionPool(object): # from_httplib will perform httplib_response.read() which will have # the side effect of letting us use this connection for another # request. - response = HTTPResponse.from_httplib(httplib_response) + response = HTTPResponse.from_httplib(httplib_response, block=block) # Put the connection back to be reused self._put_conn(conn) From a3cc97128b9511659e136afb9b3fa42ae7490bda Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 17 Sep 2011 22:46:17 -0400 Subject: [PATCH 067/255] Response.fo => Reponse.raw raise_for_status() fixes --- requests/models.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/requests/models.py b/requests/models.py index 36bba15d..2a0ead44 100644 --- a/requests/models.py +++ b/requests/models.py @@ -18,7 +18,7 @@ from datetime import datetime from .packages import urllib3 -print dir(urllib3) +# print dir(urllib3) from .config import settings from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler @@ -190,7 +190,7 @@ class Request(object): try: response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) - # response.fo = resp + response.raw = resp._raw # if self.cookiejar: @@ -202,13 +202,12 @@ class Request(object): if is_error: response.error = resp - response.url = getattr(resp, 'url', None) - return response history = [] + r = build(resp) if r.status_code in REDIRECT_STATI and not self.redirect: @@ -219,7 +218,7 @@ class Request(object): (self.allow_redirects)) ): - # r.fo.close() + r.raw.close() if not len(history) < settings.max_redirects: raise TooManyRedirects() @@ -333,10 +332,12 @@ class Request(object): body=self.data, headers=self.headers, redirect=False, - assert_same_host=False + assert_same_host=False, + block=True ) - r.socket = pool._get_conn().sock + # r.socket = pool._get_conn().sock + # r.fo = r.data # if self.cookiejar is not None: # self.cookiejar.extract_cookies(resp, req) @@ -468,10 +469,7 @@ class Response(object): self.headers = CaseInsensitiveDict() #: File-like object representation of response (for advanced usage). - self.fo = None - - #: Final URL location of Response. - self.url = None + self.raw = None #: True if no :attr:`error` occured. self.ok = False @@ -512,7 +510,7 @@ class Response(object): def generate(): while 1: - chunk = self.fo.read(chunk_size) + chunk = self.raw.read(chunk_size) if not chunk: break yield chunk @@ -540,7 +538,7 @@ class Response(object): 'already consumed') # Read the contents. - self._content = self.fo.read() + self._content = self.raw.read() # Decode GZip'd content. if 'gzip' in self.headers.get('content-encoding', ''): @@ -559,9 +557,18 @@ class Response(object): def raise_for_status(self): """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured.""" + if self.error: raise self.error + if (self.status_code >= 300) and (self.status_code < 400): + raise Exception('300 yo') + + elif (self.status_code >= 400) and (self.status_code < 500): + raise Exception('400 yo') + + elif (self.status_code >= 500) and (self.status_code < 600): + raise Exception('500 yo') class AuthManager(object): From 2d38f518348625cc9c6b8f5ee63a86bda784630e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 18 Sep 2011 01:32:00 -0400 Subject: [PATCH 068/255] kill AuthenticationError --- requests/exceptions.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/requests/exceptions.py b/requests/exceptions.py index f6585f46..c7aeca63 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -10,9 +10,6 @@ class RequestException(Exception): """There was an ambiguous exception that occured while handling your request.""" -class AuthenticationError(RequestException): - """The authentication credentials provided were invalid.""" - class Timeout(RequestException): """The request timed out.""" From 2a2e7bd19082af86f4c4ed7c00d68b99970b7bb8 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 18 Sep 2011 01:32:37 -0400 Subject: [PATCH 069/255] =?UTF-8?q?pass=20around=20connections=20like=20po?= =?UTF-8?q?k=C3=A9mon=20cards?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requests/api.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/requests/api.py b/requests/api.py index 60d0d852..a8c2f85d 100644 --- a/requests/api.py +++ b/requests/api.py @@ -23,7 +23,8 @@ __all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete') def request(method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, - timeout=None, allow_redirects=False, proxies=None, hooks=None): + timeout=None, allow_redirects=False, proxies=None, hooks=None, + _connection=None): """Constructs and sends a :class:`Request `. Returns :class:`Response ` object. @@ -39,6 +40,7 @@ def request(method, url, :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + :param _connection: (optional) An HTTP Connection to re-use. """ method = str(method).upper() @@ -64,19 +66,20 @@ def request(method, url, auth = auth, timeout = timeout or config.settings.timeout, allow_redirects = allow_redirects, - proxies = proxies or config.settings.proxies, + proxies = proxies or config.settings.proxies ) # Arguments manipulation hook. args = dispatch_hook('args', hooks, args) + # Create Request object. r = Request(**args) # Pre-request hook. r = dispatch_hook('pre_request', hooks, r) # Send the HTTP Request. - r.send() + r.send(connection=_connection) # Post-request hook. r = dispatch_hook('post_request', hooks, r) @@ -94,8 +97,8 @@ def get(url, **kwargs): :param **kwargs: Optional arguments that ``request`` takes. """ - if "allow_redirects" not in kwargs: - kwargs["allow_redirects"] = True + if 'allow_redirects' not in kwargs: + kwargs['allow_redirects'] = True return request('get', url, **kwargs) @@ -107,8 +110,8 @@ def head(url, **kwargs): :param **kwargs: Optional arguments that ``request`` takes. """ - if "allow_redirects" not in kwargs: - kwargs["allow_redirects"] = True + if 'allow_redirects' not in kwargs: + kwargs['allow_redirects'] = True return request('head', url, **kwargs) From 645933a5da8a6554045f918bfca7798bed7c4b7c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 18 Sep 2011 01:32:54 -0400 Subject: [PATCH 070/255] Gotta catch 'em all! --- requests/models.py | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/requests/models.py b/requests/models.py index 2a0ead44..43e141bf 100644 --- a/requests/models.py +++ b/requests/models.py @@ -21,13 +21,10 @@ from .packages import urllib3 # print dir(urllib3) from .config import settings -from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler from .structures import CaseInsensitiveDict -# from .packages.poster.encode import multipart_encode -# from .packages.poster.streaminghttp import register_openers, get_handlers from .utils import dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip from .status_codes import codes -from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, TooManyRedirects +from .exceptions import RequestException, Timeout, URLRequired, TooManyRedirects REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) @@ -209,6 +206,7 @@ class Request(object): r = build(resp) + r._response = resp if r.status_code in REDIRECT_STATI and not self.redirect: @@ -303,7 +301,7 @@ class Request(object): return self.url - def send(self, anyway=False): + def send(self, connection=None, anyway=False): """Sends the shit.""" # Safety check. @@ -320,20 +318,24 @@ class Request(object): # req = _Request(url, data=self._enc_data, method=self.method) - if not self.sent or anyway: + if (anyway) or (not self.sent): try: + if not connection: + connection = urllib3.connection_from_url(url, + timeout=self.timeout) + do_block = False + else: + do_block = True - pool = urllib3.connection_from_url(url, timeout=self.timeout) - - r = pool.urlopen( + r = connection.urlopen( method=self.method, url=url, body=self.data, headers=self.headers, redirect=False, assert_same_host=False, - block=True + block=do_block ) # r.socket = pool._get_conn().sock @@ -524,6 +526,7 @@ class Response(object): gen = stream_decode_response_unicode(gen, self) return gen + @property def content(self): """Content of the response, in bytes or unicode @@ -534,11 +537,13 @@ class Response(object): return self._content if self._content_consumed: - raise RuntimeError('The content for this response was ' - 'already consumed') + raise RuntimeError( + 'The content for this response was already consumed') # Read the contents. - self._content = self.raw.read() + # print self.raw.__dict__ + self._content = self.raw.read() or self._response.data + # print self.raw.__dict__ # Decode GZip'd content. if 'gzip' in self.headers.get('content-encoding', ''): @@ -556,7 +561,9 @@ class Response(object): def raise_for_status(self): - """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured.""" + """Raises stored :class:`HTTPError` or :class:`URLError`, + if one occured. + """ if self.error: raise self.error @@ -729,11 +736,11 @@ class AuthObject(object): """ _handlers = { - 'basic': HTTPBasicAuthHandler, - 'forced_basic': HTTPForcedBasicAuthHandler, - 'digest': HTTPDigestAuthHandler, - 'proxy_basic': urllib2.ProxyBasicAuthHandler, - 'proxy_digest': urllib2.ProxyDigestAuthHandler + # 'basic': HTTPBasicAuthHandler, + # 'forced_basic': HTTPForcedBasicAuthHandler, + # 'digest': HTTPDigestAuthHandler, + # 'proxy_basic': urllib2.ProxyBasicAuthHandler, + # 'proxy_digest': urllib2.ProxyDigestAuthHandler } def __init__(self, username, password, handler='forced_basic', realm=None): From 18270e1059c8eaea2bced14caad2d35e78fe7e8a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 18 Sep 2011 01:43:04 -0400 Subject: [PATCH 071/255] cleanups --- requests/models.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/requests/models.py b/requests/models.py index 43e141bf..653ae059 100644 --- a/requests/models.py +++ b/requests/models.py @@ -311,23 +311,37 @@ class Request(object): url = self._build_url() # Setup Files. + if self.files: + pass + # Setup form data. - # - # def urlopen(self, method, url, body=None, headers=None, retries=3, - # redirect=True, assert_same_host=True): - # req = _Request(url, data=self._enc_data, method=self.method) + elif self.data: + pass + + # Setup cookies. + elif self.cookies: + pass + # req = _Request(url, data=self._enc_data, method=self.method) + + # Only send the Request if new or forced. if (anyway) or (not self.sent): try: + + # Create a new HTTP connection, since one wasn't passed in. if not connection: connection = urllib3.connection_from_url(url, timeout=self.timeout) + + # One-off request. Delay fetching the content until needed. do_block = False else: + # Part of a connection pool, so no streaming. Sorry! do_block = True + # Create the connection. r = connection.urlopen( method=self.method, url=url, @@ -338,9 +352,6 @@ class Request(object): block=do_block ) - # r.socket = pool._get_conn().sock - # r.fo = r.data - # if self.cookiejar is not None: # self.cookiejar.extract_cookies(resp, req) From 7f3f641fb81463ae56d4e70b2de7daeb39e5d09b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 18 Sep 2011 01:48:23 -0400 Subject: [PATCH 072/255] super comments! --- requests/models.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index 653ae059..15614087 100644 --- a/requests/models.py +++ b/requests/models.py @@ -285,19 +285,34 @@ class Request(object): """Build the actual URL to use.""" # Support for unicode domain names and paths. - scheme, netloc, path, params, query, fragment = urlparse(self.url) + (scheme, netloc, path, params, query, fragment) = urlparse(self.url) + + # International Domain Name netloc = netloc.encode('idna') + + # Encode the path to to utf-8. if isinstance(path, unicode): path = path.encode('utf-8') - path = urllib.quote(path, safe="%/:=&?~#+!$,;'@()*[]") - self.url = str(urlunparse([ scheme, netloc, path, params, query, fragment ])) + # URL-encode the path. + path = urllib.quote(path, safe="%/:=&?~#+!$,;'@()*[]") + + # Turn it back into a bytestring. + self.url = str(urlunparse([scheme, netloc, path, params, query, fragment])) + + # Query Parameters? if self._enc_params: + + # If query parameters already exist in the URL, append. if urlparse(self.url).query: return '%s&%s' % (self.url, self._enc_params) + + # Otherwise, have at it. else: return '%s?%s' % (self.url, self._enc_params) + else: + # Kosher URL. return self.url From 332f55e552326323e205dd5069a1726439cca3cd Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 18 Sep 2011 01:59:28 -0400 Subject: [PATCH 073/255] serious commenting --- requests/models.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/requests/models.py b/requests/models.py index 15614087..daaf88f7 100644 --- a/requests/models.py +++ b/requests/models.py @@ -202,27 +202,40 @@ class Request(object): return response + # Request collector. history = [] - + # Create the lone response object. r = build(resp) + + # Store the HTTP response, just in case. r._response = resp + # It's a redirect, and we're not already in a redirect loop. if r.status_code in REDIRECT_STATI and not self.redirect: while ( + # There's a `Location` header. ('location' in r.headers) and + + # See other response. ((r.status_code is codes.see_other) or + + # Opt-in to redirects for non- idempotent methods. (self.allow_redirects)) ): + # We already redirected. Don't keep it alive. r.raw.close() - if not len(history) < settings.max_redirects: + # Woah, this is getting crazy. + if len(history) >= settings.max_redirects: raise TooManyRedirects() + # Add the old request to the history collector. history.append(r) + # Redirect to... url = r.headers['location'] # Handle redirection without scheme (see: RFC 1808 Section 4) @@ -238,23 +251,33 @@ class Request(object): parsed_url[2] = urllib.quote(parsed_url[2], safe="%/:=&?~#+!$,;'@()*[]") url = urljoin(r.url, str(urlunparse(parsed_url))) + # If 303, convert to idempotent GET. # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 if r.status_code is codes.see_other: method = 'GET' else: method = self.method + # Create the new Request. request = Request( url, self.headers, self.files, method, self.data, self.params, self.auth, self.cookiejar, + + # Flag as part of a redirect loop. redirect=True ) + + # Send her away! request.send() r = request.response + # Insert collected history. r.history = history + # Attach Response to Request. self.response = r + + # Give Response some context. self.response.request = self @@ -344,7 +367,6 @@ class Request(object): if (anyway) or (not self.sent): try: - # Create a new HTTP connection, since one wasn't passed in. if not connection: connection = urllib3.connection_from_url(url, @@ -353,7 +375,7 @@ class Request(object): # One-off request. Delay fetching the content until needed. do_block = False else: - # Part of a connection pool, so no streaming. Sorry! + # Part of a connection pool, so no fancy stuff. Sorry! do_block = True # Create the connection. @@ -367,6 +389,7 @@ class Request(object): block=do_block ) + # Extract cookies. # if self.cookiejar is not None: # self.cookiejar.extract_cookies(resp, req) From 04bcab80574c7c2f1cf8e090ca34cd44a96b1e64 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 18 Sep 2011 03:20:11 -0400 Subject: [PATCH 074/255] fails --- requests/models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index daaf88f7..4834fe32 100644 --- a/requests/models.py +++ b/requests/models.py @@ -357,9 +357,8 @@ class Request(object): pass # Setup cookies. - elif self.cookies: - pass - + # elif self.cookies: + # pass # req = _Request(url, data=self._enc_data, method=self.method) From 95b0549c75c61fe0b759400713112de357ae74b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Thu, 22 Sep 2011 11:13:51 +0200 Subject: [PATCH 075/255] Ignore unknown encoding. --- requests/utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 17f79a8a..22084d05 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -181,12 +181,12 @@ def unicode_from_html(content): def stream_decode_response_unicode(iterator, r): """Stream decodes a iterator.""" encoding = get_encoding_from_headers(r.headers) - if encoding is None: + try: + decoder = codecs.getincrementaldecoder(str(encoding))(errors='replace') + except LookupError: for item in iterator: yield item return - - decoder = codecs.getincrementaldecoder(encoding)(errors='replace') for chunk in iterator: rv = decoder.decode(chunk) if rv: @@ -221,6 +221,8 @@ def get_unicode_from_response(r): return unicode(r.content, encoding) except UnicodeError: tried_encodings.append(encoding) + except LookupError: + return r.content # Fall back: try: From 5345a89dc5121daffddcdab008ddfb324d6b87c8 Mon Sep 17 00:00:00 2001 From: Michael Van Veen Date: Thu, 22 Sep 2011 03:38:58 -0700 Subject: [PATCH 076/255] added toa AUTHORS file --- AUTHORS | 3 +- requests/api.py | 22 +++++++------- requests/config.py | 15 ++++++++++ requests/core.py | 2 +- requests/exceptions.py | 8 ++++- requests/models.py | 63 ++++++++++++++-------------------------- requests/monkeys.py | 14 +++------ requests/sessions.py | 6 +--- requests/status_codes.py | 2 +- requests/structures.py | 4 ++- requests/utils.py | 15 +++++----- 11 files changed, 74 insertions(+), 80 deletions(-) diff --git a/AUTHORS b/AUTHORS index af00b19e..0478a751 100644 --- a/AUTHORS +++ b/AUTHORS @@ -43,5 +43,6 @@ Patches and Suggestions - Rick Mak - Johan Bergström - Josselin Jacquard +- Michael Van Veen - Mike Waldner -- Serge Domkowski \ No newline at end of file +- Serge Domkowski diff --git a/requests/api.py b/requests/api.py index 60d0d852..48823a9a 100644 --- a/requests/api.py +++ b/requests/api.py @@ -54,17 +54,17 @@ def request(method, url, headers[k] = header_expand(v) args = dict( - method = method, - url = url, - data = data, - params = params, - headers = headers, - cookiejar = cookies, - files = files, - auth = auth, - timeout = timeout or config.settings.timeout, - allow_redirects = allow_redirects, - proxies = proxies or config.settings.proxies, + method=method, + url=url, + data=data, + params=params, + headers=headers, + cookiejar=cookies, + files=files, + auth=auth, + timeout=timeout or config.settings.timeout, + allow_redirects=allow_redirects, + proxies=proxies or config.settings.proxies, ) # Arguments manipulation hook. diff --git a/requests/config.py b/requests/config.py index 676a5f0e..55c1b88c 100644 --- a/requests/config.py +++ b/requests/config.py @@ -6,8 +6,23 @@ requests.config This module provides the Requests settings feature set. +settings parameters: + +TODO: Verify!!! +TODO: Make sure format is acceptabl/cool +- :base_headers: - Sets default User-Agent to `python-requests.org` +- :accept_gzip: - Whether or not to accept gzip-compressed data +- :proxies: - http proxies? +- :verbose: - display verbose information? +- :timeout: - timeout time until request terminates +- :max_redirects: - maximum number of allowed redirects? +- :decode_unicode: - whether or not to accept unicode? + +Used globally + """ + class Settings(object): def __init__(self, **kwargs): diff --git a/requests/core.py b/requests/core.py index e1ba1853..8f765ed1 100644 --- a/requests/core.py +++ b/requests/core.py @@ -26,4 +26,4 @@ from sessions import session from status_codes import codes from config import settings -import utils \ No newline at end of file +import utils diff --git a/requests/exceptions.py b/requests/exceptions.py index c08c6148..3a650270 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -6,21 +6,27 @@ requests.exceptions """ + class RequestException(Exception): """There was an ambiguous exception that occured while handling your request.""" + class AuthenticationError(RequestException): """The authentication credentials provided were invalid.""" - + + class Timeout(RequestException): """The request timed out.""" + class URLRequired(RequestException): """A valid URL is required to make a request.""" + class InvalidMethod(RequestException): """An inappropriate method was attempted.""" + class TooManyRedirects(RequestException): """Too many redirects.""" diff --git a/requests/models.py b/requests/models.py index 742acf2c..f6d84ebd 100644 --- a/requests/models.py +++ b/requests/models.py @@ -30,7 +30,6 @@ from .exceptions import RequestException, AuthenticationError, Timeout, URLRequi REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) - class Request(object): """The :class:`Request ` object. It carries out all functionality of Requests. Recommended interface is with the Requests functions. @@ -96,7 +95,6 @@ class Request(object): #: True if Request has been sent. self.sent = False - # Header manipulation and defaults. if settings.accept_gzip: @@ -113,18 +111,15 @@ class Request(object): self.headers = headers - def __repr__(self): return '' % (self.method) - def _checks(self): """Deterministic checks for consistency.""" if not self.url: raise URLRequired - def _get_opener(self): """Creates appropriate opener object for urllib2.""" @@ -173,13 +168,11 @@ class Request(object): return opener.open - def _build_response(self, resp, is_error=False): """Build internal :class:`Response ` object from given response. """ - def build(resp): response = Response() @@ -193,7 +186,6 @@ class Request(object): response.cookies = dict_from_cookiejar(self.cookiejar) - except AttributeError: pass @@ -204,7 +196,6 @@ class Request(object): return response - history = [] r = build(resp) @@ -258,7 +249,6 @@ class Request(object): self.response = r self.response.request = self - @staticmethod def _encode_params(data): """Encode parameters in a piece of data. @@ -276,22 +266,28 @@ class Request(object): for k, vs in data.items(): for v in isinstance(vs, list) and vs or [vs]: result.append((k.encode('utf-8') if isinstance(k, unicode) else k, - v.encode('utf-8') if isinstance(v, unicode) else v)) - return result, urllib.urlencode(result, doseq=True) + v.encode('utf-8') if isinstance(v, unicode) else v) + ) + return (result, urllib.urlencode(result, doseq=True)) + else: return data, data - def _build_url(self): """Build the actual URL to use.""" # Support for unicode domain names and paths. scheme, netloc, path, params, query, fragment = urlparse(self.url) netloc = netloc.encode('idna') + if isinstance(path, unicode): path = path.encode('utf-8') + path = urllib.quote(path, safe="%/:=&?~#+!$,;'@()*[]") - self.url = str(urlunparse([ scheme, netloc, path, params, query, fragment ])) + + self.url = str(urlunparse( + [scheme, netloc, path, params, query, fragment] + )) if self._enc_params: if urlparse(self.url).query: @@ -301,7 +297,6 @@ class Request(object): else: return self.url - def send(self, anyway=False): """Sends the request. Returns True of successful, false if not. If there was an HTTPError during transmission, @@ -321,7 +316,6 @@ class Request(object): datetime.now().isoformat(), self.method, self.url )) - url = self._build_url() if self.method in ('GET', 'HEAD', 'DELETE'): req = _Request(url, method=self.method) @@ -340,7 +334,7 @@ class Request(object): req = _Request(url, data=self._enc_data, method=self.method) if self.headers: - for k,v in self.headers.iteritems(): + for k, v in self.headers.iteritems(): req.add_header(k, v) if not self.sent or anyway: @@ -381,17 +375,17 @@ class Request(object): self._build_response(resp) self.response.ok = True - self.sent = self.response.ok return self.sent class Response(object): - """The core :class:`Response ` object. All - :class:`Request ` objects contain a - :class:`response ` attribute, which is an instance - of this class. + """The core :class:`Response ` object. + + + All :class:`Request ` objects contain a :class:`response + ` attribute, which is an instance of this class. """ def __init__(self): @@ -430,11 +424,9 @@ class Response(object): #: A dictionary of Cookies the server sent back. self.cookies = None - def __repr__(self): return '' % (self.status_code) - def __nonzero__(self): """Returns true if :attr:`status_code` is 'OK'.""" @@ -496,14 +488,12 @@ class Response(object): self._content_consumed = True return self._content - def raise_for_status(self): """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured.""" if self.error: raise self.error - class AuthManager(object): """Requests Authentication Manager.""" @@ -516,16 +506,13 @@ class AuthManager(object): return singleton - def __init__(self): self.passwd = {} self._auth = {} - def __repr__(self): return '' % (self.method) - def add_auth(self, uri, auth): """Registers AuthObject to AuthManager.""" @@ -540,7 +527,6 @@ class AuthManager(object): self._auth[uri] = auth - def add_password(self, realm, uri, user, passwd): """Adds password to AuthManager.""" # uri could be a single URI or a sequence @@ -553,7 +539,6 @@ class AuthManager(object): self.passwd[reduced_uri] = {} self.passwd[reduced_uri] = (user, passwd) - def find_user_password(self, realm, authuri): for uris, authinfo in self.passwd.iteritems(): reduced_authuri = self.reduce_uri(authuri, False) @@ -563,7 +548,6 @@ class AuthManager(object): return (None, None) - def get_auth(self, uri): (in_domain, in_path) = self.reduce_uri(uri, False) @@ -574,7 +558,6 @@ class AuthManager(object): if path in in_path: return authority - def reduce_uri(self, uri, default_port=True): """Accept authority or URI and extract only the authority and path.""" @@ -603,7 +586,6 @@ class AuthManager(object): return authority, path - def is_suburi(self, base, test): """Check if test is below base in a URI tree @@ -618,11 +600,9 @@ class AuthManager(object): return True return False - def empty(self): self.passwd = {} - def remove(self, uri, realm=None): # uri could be a single URI or a sequence if isinstance(uri, basestring): @@ -632,7 +612,6 @@ class AuthManager(object): reduced_uri = tuple([self.reduce_uri(u, default_port) for u in uri]) del self.passwd[reduced_uri][realm] - def __contains__(self, uri): # uri could be a single URI or a sequence if isinstance(uri, basestring): @@ -648,12 +627,12 @@ class AuthManager(object): auth_manager = AuthManager() - class AuthObject(object): - """The :class:`AuthObject` is a simple HTTP Authentication token. When - given to a Requests function, it enables Basic HTTP Authentication for that - Request. You can also enable Authorization for domain realms with AutoAuth. - See AutoAuth for more details. + """The :class:`AuthObject` is a simple HTTP Authentication token. + + When given to a Requests function, it enables Basic HTTP Authentication + for that Request. You can also enable Authorization for domain realms + with AutoAuth. See AutoAuth for more details. :param username: Username to authenticate with. :param password: Password for given username. diff --git a/requests/monkeys.py b/requests/monkeys.py index c8380711..8f10d29f 100644 --- a/requests/monkeys.py +++ b/requests/monkeys.py @@ -8,8 +8,9 @@ Urllib2 Monkey patches. """ -import urllib2 import re +import urllib2 + class Request(urllib2.Request): """Hidden wrapper around the urllib2.Request object. Allows for manual @@ -35,7 +36,6 @@ class HTTPRedirectHandler(urllib2.HTTPRedirectHandler): http_error_302 = http_error_303 = http_error_307 = http_error_301 - class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler): """HTTP Basic Auth Handler with authentication loop fixes.""" @@ -44,14 +44,12 @@ class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler): self.retried_req = None self.retried = 0 - def reset_retry_count(self): # Python 2.6.5 will call this on 401 or 407 errors and thus loop # forever. We disable reset_retry_count completely and reset in # http_error_auth_reqed instead. pass - def http_error_auth_reqed(self, auth_header, host, req, headers): # Reset the retry counter once for each request. if req is not self.retried_req: @@ -63,7 +61,6 @@ class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler): ) - class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler): """HTTP Basic Auth Handler with forced Authentication.""" @@ -71,10 +68,9 @@ class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler): rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+' 'realm=(["\'])(.*?)\\2', re.I) - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs): HTTPBasicAuthHandler.__init__(self, *args, **kwargs) - def http_error_401(self, req, fp, code, msg, headers): url = req.get_full_url() response = self._http_error_auth_reqed('www-authenticate', url, req, headers) @@ -83,7 +79,6 @@ class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler): http_error_404 = http_error_401 - def _http_error_auth_reqed(self, authreq, host, req, headers): authreq = headers.get(authreq, None) @@ -116,7 +111,6 @@ class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler): return response - class HTTPDigestAuthHandler(urllib2.HTTPDigestAuthHandler): def __init__(self, *args, **kwargs): @@ -145,4 +139,4 @@ class HTTPDigestAuthHandler(urllib2.HTTPDigestAuthHandler): arg = inst.args[0] if arg.startswith("AbstractDigestAuthHandler doesn't know "): return - raise \ No newline at end of file + raise diff --git a/requests/sessions.py b/requests/sessions.py index 50b09f61..7145d278 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -15,13 +15,11 @@ from . import api from .utils import add_dict_to_cookiejar - class Session(object): """A Requests session.""" __attrs__ = ['headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks'] - def __init__(self, **kwargs): # Set up a CookieJar to be used by default @@ -34,7 +32,6 @@ class Session(object): # Map and wrap requests.api methods self._map_api_methods() - def __repr__(self): return '' % (id(self)) @@ -45,7 +42,6 @@ class Session(object): # print args pass - def _map_api_methods(self): """Reads each available method from requests.api and decorates them with a wrapper, which inserts any instance-local attributes @@ -81,4 +77,4 @@ class Session(object): def session(**kwargs): """Returns a :class:`Session` for context-managment.""" - return Session(**kwargs) \ No newline at end of file + return Session(**kwargs) diff --git a/requests/status_codes.py b/requests/status_codes.py index a809de6a..025dc28f 100644 --- a/requests/status_codes.py +++ b/requests/status_codes.py @@ -80,4 +80,4 @@ for (code, titles) in _codes.items(): for title in titles: setattr(codes, title, code) if not title.startswith('\\'): - setattr(codes, title.upper(), code) \ No newline at end of file + setattr(codes, title.upper(), code) diff --git a/requests/structures.py b/requests/structures.py index d068bf9c..b16e75df 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -8,6 +8,7 @@ Datastructures that power Requests. """ + class CaseInsensitiveDict(dict): """Case-insensitive Dictionary @@ -46,6 +47,7 @@ class CaseInsensitiveDict(dict): else: return default + class LookupDict(dict): """Dictionary lookup object.""" @@ -62,4 +64,4 @@ class LookupDict(dict): return self.__dict__.get(key, None) def get(self, key, default=None): - return self.__dict__.get(key, default) \ No newline at end of file + return self.__dict__.get(key, default) diff --git a/requests/utils.py b/requests/utils.py index 17f79a8a..7a5c6d70 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -51,10 +51,9 @@ def header_expand(headers): collector.append('; '.join(_params)) - if not len(headers) == i+1: + if not len(headers) == i + 1: collector.append(', ') - # Remove trailing seperators. if collector[-1] in (', ', '; '): del collector[-1] @@ -62,7 +61,6 @@ def header_expand(headers): return ''.join(collector) - def dict_from_cookiejar(cj): """Returns a key/value dictionary from a CookieJar. @@ -235,8 +233,8 @@ def decode_gzip(content): :param content: bytestring to gzip-decode. """ - return zlib.decompress(content, 16+zlib.MAX_WBITS) - return zlib.decompress(content, 16+zlib.MAX_WBITS) + return zlib.decompress(content, 16 + zlib.MAX_WBITS) + return zlib.decompress(content, 16 + zlib.MAX_WBITS) return zlib.decompress(content, 16 + zlib.MAX_WBITS) @@ -255,6 +253,7 @@ def stream_decode_gzip(iterator): except zlib.error: pass + def curl_from_request(request): """Returns a curl command from the request. @@ -276,10 +275,13 @@ def curl_from_request(request): #: -u/--user - Specify the user name and password to use for server auth. #: Basic Auth only for now auth = '' + if request.auth is not None: - auth = '-u "%s:%s" ' % (request.auth.username, request.auth.password) + + auth = '-u "%s:%s" ' % (request.auth.username, request.auth.password) method = '' + if request.method.upper() == 'HEAD': #: -I/--head - fetch headers only. method = '-I ' @@ -321,4 +323,3 @@ def curl_from_request(request): #: Params handled in _build_url return curl + auth + method + header + cookies + form + '"' + request._build_url() + '"' - From dcebab4c35369f9f91dab3eb6ff877509f7d38c2 Mon Sep 17 00:00:00 2001 From: Michael Van Veen Date: Thu, 22 Sep 2011 03:38:58 -0700 Subject: [PATCH 077/255] PEP8fied all the things I ran the pep8 checker (version 0.6.1) on the code base and tried to clean up as many pep8 errors as I could. Everything except line-width errors are gone (I was worried about messing up the documentor since I haven't fiddled with it). --- AUTHORS | 3 +- requests/api.py | 22 +++++++------- requests/config.py | 15 ++++++++++ requests/core.py | 2 +- requests/exceptions.py | 8 ++++- requests/models.py | 63 ++++++++++++++-------------------------- requests/monkeys.py | 14 +++------ requests/sessions.py | 6 +--- requests/status_codes.py | 2 +- requests/structures.py | 4 ++- requests/utils.py | 15 +++++----- 11 files changed, 74 insertions(+), 80 deletions(-) diff --git a/AUTHORS b/AUTHORS index af00b19e..0478a751 100644 --- a/AUTHORS +++ b/AUTHORS @@ -43,5 +43,6 @@ Patches and Suggestions - Rick Mak - Johan Bergström - Josselin Jacquard +- Michael Van Veen - Mike Waldner -- Serge Domkowski \ No newline at end of file +- Serge Domkowski diff --git a/requests/api.py b/requests/api.py index 60d0d852..48823a9a 100644 --- a/requests/api.py +++ b/requests/api.py @@ -54,17 +54,17 @@ def request(method, url, headers[k] = header_expand(v) args = dict( - method = method, - url = url, - data = data, - params = params, - headers = headers, - cookiejar = cookies, - files = files, - auth = auth, - timeout = timeout or config.settings.timeout, - allow_redirects = allow_redirects, - proxies = proxies or config.settings.proxies, + method=method, + url=url, + data=data, + params=params, + headers=headers, + cookiejar=cookies, + files=files, + auth=auth, + timeout=timeout or config.settings.timeout, + allow_redirects=allow_redirects, + proxies=proxies or config.settings.proxies, ) # Arguments manipulation hook. diff --git a/requests/config.py b/requests/config.py index 676a5f0e..55c1b88c 100644 --- a/requests/config.py +++ b/requests/config.py @@ -6,8 +6,23 @@ requests.config This module provides the Requests settings feature set. +settings parameters: + +TODO: Verify!!! +TODO: Make sure format is acceptabl/cool +- :base_headers: - Sets default User-Agent to `python-requests.org` +- :accept_gzip: - Whether or not to accept gzip-compressed data +- :proxies: - http proxies? +- :verbose: - display verbose information? +- :timeout: - timeout time until request terminates +- :max_redirects: - maximum number of allowed redirects? +- :decode_unicode: - whether or not to accept unicode? + +Used globally + """ + class Settings(object): def __init__(self, **kwargs): diff --git a/requests/core.py b/requests/core.py index e1ba1853..8f765ed1 100644 --- a/requests/core.py +++ b/requests/core.py @@ -26,4 +26,4 @@ from sessions import session from status_codes import codes from config import settings -import utils \ No newline at end of file +import utils diff --git a/requests/exceptions.py b/requests/exceptions.py index c08c6148..3a650270 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -6,21 +6,27 @@ requests.exceptions """ + class RequestException(Exception): """There was an ambiguous exception that occured while handling your request.""" + class AuthenticationError(RequestException): """The authentication credentials provided were invalid.""" - + + class Timeout(RequestException): """The request timed out.""" + class URLRequired(RequestException): """A valid URL is required to make a request.""" + class InvalidMethod(RequestException): """An inappropriate method was attempted.""" + class TooManyRedirects(RequestException): """Too many redirects.""" diff --git a/requests/models.py b/requests/models.py index 742acf2c..f6d84ebd 100644 --- a/requests/models.py +++ b/requests/models.py @@ -30,7 +30,6 @@ from .exceptions import RequestException, AuthenticationError, Timeout, URLRequi REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) - class Request(object): """The :class:`Request ` object. It carries out all functionality of Requests. Recommended interface is with the Requests functions. @@ -96,7 +95,6 @@ class Request(object): #: True if Request has been sent. self.sent = False - # Header manipulation and defaults. if settings.accept_gzip: @@ -113,18 +111,15 @@ class Request(object): self.headers = headers - def __repr__(self): return '' % (self.method) - def _checks(self): """Deterministic checks for consistency.""" if not self.url: raise URLRequired - def _get_opener(self): """Creates appropriate opener object for urllib2.""" @@ -173,13 +168,11 @@ class Request(object): return opener.open - def _build_response(self, resp, is_error=False): """Build internal :class:`Response ` object from given response. """ - def build(resp): response = Response() @@ -193,7 +186,6 @@ class Request(object): response.cookies = dict_from_cookiejar(self.cookiejar) - except AttributeError: pass @@ -204,7 +196,6 @@ class Request(object): return response - history = [] r = build(resp) @@ -258,7 +249,6 @@ class Request(object): self.response = r self.response.request = self - @staticmethod def _encode_params(data): """Encode parameters in a piece of data. @@ -276,22 +266,28 @@ class Request(object): for k, vs in data.items(): for v in isinstance(vs, list) and vs or [vs]: result.append((k.encode('utf-8') if isinstance(k, unicode) else k, - v.encode('utf-8') if isinstance(v, unicode) else v)) - return result, urllib.urlencode(result, doseq=True) + v.encode('utf-8') if isinstance(v, unicode) else v) + ) + return (result, urllib.urlencode(result, doseq=True)) + else: return data, data - def _build_url(self): """Build the actual URL to use.""" # Support for unicode domain names and paths. scheme, netloc, path, params, query, fragment = urlparse(self.url) netloc = netloc.encode('idna') + if isinstance(path, unicode): path = path.encode('utf-8') + path = urllib.quote(path, safe="%/:=&?~#+!$,;'@()*[]") - self.url = str(urlunparse([ scheme, netloc, path, params, query, fragment ])) + + self.url = str(urlunparse( + [scheme, netloc, path, params, query, fragment] + )) if self._enc_params: if urlparse(self.url).query: @@ -301,7 +297,6 @@ class Request(object): else: return self.url - def send(self, anyway=False): """Sends the request. Returns True of successful, false if not. If there was an HTTPError during transmission, @@ -321,7 +316,6 @@ class Request(object): datetime.now().isoformat(), self.method, self.url )) - url = self._build_url() if self.method in ('GET', 'HEAD', 'DELETE'): req = _Request(url, method=self.method) @@ -340,7 +334,7 @@ class Request(object): req = _Request(url, data=self._enc_data, method=self.method) if self.headers: - for k,v in self.headers.iteritems(): + for k, v in self.headers.iteritems(): req.add_header(k, v) if not self.sent or anyway: @@ -381,17 +375,17 @@ class Request(object): self._build_response(resp) self.response.ok = True - self.sent = self.response.ok return self.sent class Response(object): - """The core :class:`Response ` object. All - :class:`Request ` objects contain a - :class:`response ` attribute, which is an instance - of this class. + """The core :class:`Response ` object. + + + All :class:`Request ` objects contain a :class:`response + ` attribute, which is an instance of this class. """ def __init__(self): @@ -430,11 +424,9 @@ class Response(object): #: A dictionary of Cookies the server sent back. self.cookies = None - def __repr__(self): return '' % (self.status_code) - def __nonzero__(self): """Returns true if :attr:`status_code` is 'OK'.""" @@ -496,14 +488,12 @@ class Response(object): self._content_consumed = True return self._content - def raise_for_status(self): """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured.""" if self.error: raise self.error - class AuthManager(object): """Requests Authentication Manager.""" @@ -516,16 +506,13 @@ class AuthManager(object): return singleton - def __init__(self): self.passwd = {} self._auth = {} - def __repr__(self): return '' % (self.method) - def add_auth(self, uri, auth): """Registers AuthObject to AuthManager.""" @@ -540,7 +527,6 @@ class AuthManager(object): self._auth[uri] = auth - def add_password(self, realm, uri, user, passwd): """Adds password to AuthManager.""" # uri could be a single URI or a sequence @@ -553,7 +539,6 @@ class AuthManager(object): self.passwd[reduced_uri] = {} self.passwd[reduced_uri] = (user, passwd) - def find_user_password(self, realm, authuri): for uris, authinfo in self.passwd.iteritems(): reduced_authuri = self.reduce_uri(authuri, False) @@ -563,7 +548,6 @@ class AuthManager(object): return (None, None) - def get_auth(self, uri): (in_domain, in_path) = self.reduce_uri(uri, False) @@ -574,7 +558,6 @@ class AuthManager(object): if path in in_path: return authority - def reduce_uri(self, uri, default_port=True): """Accept authority or URI and extract only the authority and path.""" @@ -603,7 +586,6 @@ class AuthManager(object): return authority, path - def is_suburi(self, base, test): """Check if test is below base in a URI tree @@ -618,11 +600,9 @@ class AuthManager(object): return True return False - def empty(self): self.passwd = {} - def remove(self, uri, realm=None): # uri could be a single URI or a sequence if isinstance(uri, basestring): @@ -632,7 +612,6 @@ class AuthManager(object): reduced_uri = tuple([self.reduce_uri(u, default_port) for u in uri]) del self.passwd[reduced_uri][realm] - def __contains__(self, uri): # uri could be a single URI or a sequence if isinstance(uri, basestring): @@ -648,12 +627,12 @@ class AuthManager(object): auth_manager = AuthManager() - class AuthObject(object): - """The :class:`AuthObject` is a simple HTTP Authentication token. When - given to a Requests function, it enables Basic HTTP Authentication for that - Request. You can also enable Authorization for domain realms with AutoAuth. - See AutoAuth for more details. + """The :class:`AuthObject` is a simple HTTP Authentication token. + + When given to a Requests function, it enables Basic HTTP Authentication + for that Request. You can also enable Authorization for domain realms + with AutoAuth. See AutoAuth for more details. :param username: Username to authenticate with. :param password: Password for given username. diff --git a/requests/monkeys.py b/requests/monkeys.py index c8380711..8f10d29f 100644 --- a/requests/monkeys.py +++ b/requests/monkeys.py @@ -8,8 +8,9 @@ Urllib2 Monkey patches. """ -import urllib2 import re +import urllib2 + class Request(urllib2.Request): """Hidden wrapper around the urllib2.Request object. Allows for manual @@ -35,7 +36,6 @@ class HTTPRedirectHandler(urllib2.HTTPRedirectHandler): http_error_302 = http_error_303 = http_error_307 = http_error_301 - class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler): """HTTP Basic Auth Handler with authentication loop fixes.""" @@ -44,14 +44,12 @@ class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler): self.retried_req = None self.retried = 0 - def reset_retry_count(self): # Python 2.6.5 will call this on 401 or 407 errors and thus loop # forever. We disable reset_retry_count completely and reset in # http_error_auth_reqed instead. pass - def http_error_auth_reqed(self, auth_header, host, req, headers): # Reset the retry counter once for each request. if req is not self.retried_req: @@ -63,7 +61,6 @@ class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler): ) - class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler): """HTTP Basic Auth Handler with forced Authentication.""" @@ -71,10 +68,9 @@ class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler): rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+' 'realm=(["\'])(.*?)\\2', re.I) - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs): HTTPBasicAuthHandler.__init__(self, *args, **kwargs) - def http_error_401(self, req, fp, code, msg, headers): url = req.get_full_url() response = self._http_error_auth_reqed('www-authenticate', url, req, headers) @@ -83,7 +79,6 @@ class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler): http_error_404 = http_error_401 - def _http_error_auth_reqed(self, authreq, host, req, headers): authreq = headers.get(authreq, None) @@ -116,7 +111,6 @@ class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler): return response - class HTTPDigestAuthHandler(urllib2.HTTPDigestAuthHandler): def __init__(self, *args, **kwargs): @@ -145,4 +139,4 @@ class HTTPDigestAuthHandler(urllib2.HTTPDigestAuthHandler): arg = inst.args[0] if arg.startswith("AbstractDigestAuthHandler doesn't know "): return - raise \ No newline at end of file + raise diff --git a/requests/sessions.py b/requests/sessions.py index 50b09f61..7145d278 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -15,13 +15,11 @@ from . import api from .utils import add_dict_to_cookiejar - class Session(object): """A Requests session.""" __attrs__ = ['headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks'] - def __init__(self, **kwargs): # Set up a CookieJar to be used by default @@ -34,7 +32,6 @@ class Session(object): # Map and wrap requests.api methods self._map_api_methods() - def __repr__(self): return '' % (id(self)) @@ -45,7 +42,6 @@ class Session(object): # print args pass - def _map_api_methods(self): """Reads each available method from requests.api and decorates them with a wrapper, which inserts any instance-local attributes @@ -81,4 +77,4 @@ class Session(object): def session(**kwargs): """Returns a :class:`Session` for context-managment.""" - return Session(**kwargs) \ No newline at end of file + return Session(**kwargs) diff --git a/requests/status_codes.py b/requests/status_codes.py index a809de6a..025dc28f 100644 --- a/requests/status_codes.py +++ b/requests/status_codes.py @@ -80,4 +80,4 @@ for (code, titles) in _codes.items(): for title in titles: setattr(codes, title, code) if not title.startswith('\\'): - setattr(codes, title.upper(), code) \ No newline at end of file + setattr(codes, title.upper(), code) diff --git a/requests/structures.py b/requests/structures.py index d068bf9c..b16e75df 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -8,6 +8,7 @@ Datastructures that power Requests. """ + class CaseInsensitiveDict(dict): """Case-insensitive Dictionary @@ -46,6 +47,7 @@ class CaseInsensitiveDict(dict): else: return default + class LookupDict(dict): """Dictionary lookup object.""" @@ -62,4 +64,4 @@ class LookupDict(dict): return self.__dict__.get(key, None) def get(self, key, default=None): - return self.__dict__.get(key, default) \ No newline at end of file + return self.__dict__.get(key, default) diff --git a/requests/utils.py b/requests/utils.py index 17f79a8a..7a5c6d70 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -51,10 +51,9 @@ def header_expand(headers): collector.append('; '.join(_params)) - if not len(headers) == i+1: + if not len(headers) == i + 1: collector.append(', ') - # Remove trailing seperators. if collector[-1] in (', ', '; '): del collector[-1] @@ -62,7 +61,6 @@ def header_expand(headers): return ''.join(collector) - def dict_from_cookiejar(cj): """Returns a key/value dictionary from a CookieJar. @@ -235,8 +233,8 @@ def decode_gzip(content): :param content: bytestring to gzip-decode. """ - return zlib.decompress(content, 16+zlib.MAX_WBITS) - return zlib.decompress(content, 16+zlib.MAX_WBITS) + return zlib.decompress(content, 16 + zlib.MAX_WBITS) + return zlib.decompress(content, 16 + zlib.MAX_WBITS) return zlib.decompress(content, 16 + zlib.MAX_WBITS) @@ -255,6 +253,7 @@ def stream_decode_gzip(iterator): except zlib.error: pass + def curl_from_request(request): """Returns a curl command from the request. @@ -276,10 +275,13 @@ def curl_from_request(request): #: -u/--user - Specify the user name and password to use for server auth. #: Basic Auth only for now auth = '' + if request.auth is not None: - auth = '-u "%s:%s" ' % (request.auth.username, request.auth.password) + + auth = '-u "%s:%s" ' % (request.auth.username, request.auth.password) method = '' + if request.method.upper() == 'HEAD': #: -I/--head - fetch headers only. method = '-I ' @@ -321,4 +323,3 @@ def curl_from_request(request): #: Params handled in _build_url return curl + auth + method + header + cookies + form + '"' + request._build_url() + '"' - From c48cf2bba8aafa50ae5cd77e7ab4bb476dca086c Mon Sep 17 00:00:00 2001 From: Michael Van Veen Date: Thu, 22 Sep 2011 03:52:05 -0700 Subject: [PATCH 078/255] rearranged imports in core.py --- requests/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/core.py b/requests/core.py index 8f765ed1..c9f10f55 100644 --- a/requests/core.py +++ b/requests/core.py @@ -19,11 +19,11 @@ __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' -from models import HTTPError, Request, Response from api import * +from config import settings from exceptions import * +from models import HTTPError, Request, Response from sessions import session from status_codes import codes -from config import settings import utils From cd01aa466621932dd9a00551b1550d61eab6c4da Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 23 Sep 2011 03:23:19 -0400 Subject: [PATCH 079/255] Work for Pie in docs --- docs/index.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 15f38743..63ef0669 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -41,7 +41,8 @@ Testimonials `Twitter, Inc `_, a U.S. Federal Institution, -and `Readability `_ +`Readability `_, and +`Work for Pie `_ use Requests internally. **Daniel Greenfeld** From df66fa7975dd359c97161848a175122a5406ee06 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 16:02:22 -0400 Subject: [PATCH 080/255] merge bugs --- requests/api.py | 22 ---------------------- requests/models.py | 4 ++-- 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/requests/api.py b/requests/api.py index 12381ce3..fcd02b56 100644 --- a/requests/api.py +++ b/requests/api.py @@ -56,28 +56,6 @@ def request(method, url, headers[k] = header_expand(v) args = dict( - method = method, - url = url, - data = data, - params = params, - headers = headers, - cookiejar = cookies, - files = files, - auth = auth, - timeout = timeout or config.settings.timeout, - allow_redirects = allow_redirects, - proxies = proxies or config.settings.proxies, - method = method, - url = url, - data = data, - params = params, - headers = headers, - cookiejar = cookies, - files = files, - auth = auth, - timeout = timeout or config.settings.timeout, - allow_redirects = allow_redirects, - proxies = proxies or config.settings.proxies method=method, url=url, data=data, diff --git a/requests/models.py b/requests/models.py index a7a8c815..a90924db 100644 --- a/requests/models.py +++ b/requests/models.py @@ -184,12 +184,12 @@ class Request(object): # if self.cookiejar: - response.cookies = dict_from_cookiejar(self.cookiejar) + # response.cookies = dict_from_cookiejar(self.cookiejar) # response.cookies = dict_from_cookiejar(self.cookiejar) - response.cookies = dict_from_cookiejar(self.cookiejar) + # response.cookies = dict_from_cookiejar(self.cookiejar) except AttributeError: pass From 0db4dab03e37f41827ef9fa713c3ec95876a3628 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 16:05:14 -0400 Subject: [PATCH 081/255] urllib3 updates --- requests/packages/urllib3/__init__.py | 24 +- requests/packages/urllib3/_collections.py | 115 +++++++++ requests/packages/urllib3/connectionpool.py | 244 +++++++------------- requests/packages/urllib3/exceptions.py | 29 +++ requests/packages/urllib3/filepost.py | 16 +- requests/packages/urllib3/poolmanager.py | 64 +++++ requests/packages/urllib3/response.py | 173 ++++++++++++++ 7 files changed, 492 insertions(+), 173 deletions(-) create mode 100644 requests/packages/urllib3/_collections.py create mode 100644 requests/packages/urllib3/exceptions.py create mode 100644 requests/packages/urllib3/poolmanager.py create mode 100644 requests/packages/urllib3/response.py diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index c7ade88e..19a62391 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -2,21 +2,25 @@ urllib3 - Thread-safe connection pooling and re-using. """ -from connectionpool import ( - connection_from_url, - get_host, +__author__ = "Andrey Petrov (andrey.petrov@shazow.net)" +__license__ = "MIT" +__version__ = "$Rev$" + + +from .connectionpool import ( HTTPConnectionPool, HTTPSConnectionPool, + connection_from_url, + get_host, make_headers) -# Possible exceptions -from connectionpool import ( + + +from .exceptions import ( HTTPError, MaxRetryError, SSLError, TimeoutError) -from filepost import encode_multipart_formdata - -__author__ = "Andrey Petrov (andrey.petrov@shazow.net)" -__license__ = "MIT" -__version__ = "$Rev$" +from .poolmanager import PoolManager +from .response import HTTPResponse +from .filepost import encode_multipart_formdata diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py new file mode 100644 index 00000000..0e1c8e69 --- /dev/null +++ b/requests/packages/urllib3/_collections.py @@ -0,0 +1,115 @@ +from collections import MutableMapping, deque + + +__all__ = ['RecentlyUsedContainer'] + + +class AccessEntry(object): + __slots__ = ('key', 'is_valid') + + def __init__(self, key, is_valid=True): + self.key = key + self.is_valid = is_valid + + +class RecentlyUsedContainer(MutableMapping): + """ + Provides a dict-like that maintains up to ``maxsize`` keys while throwing + away the least-recently-used keys beyond ``maxsize``. + """ + + # TODO: Make this threadsafe. _prune_invalidated_entries should be the + # only real pain-point for this. + + # If len(self.access_log) exceeds self._maxsize * CLEANUP_FACTOR, then we + # will attempt to cleanup the invalidated entries in the access_log + # datastructure during the next 'get' operation. + CLEANUP_FACTOR = 10 + + def __init__(self, maxsize=10): + self._maxsize = maxsize + + self._container = {} + + # We use a deque to to store our keys ordered by the last access. + self.access_log = deque() + + # We look up the access log entry by the key to invalidate it so we can + # insert a new authorative entry at the head without having to dig and + # find the old entry for removal immediately. + self.access_lookup = {} + + # Trigger a heap cleanup when we get past this size + self.access_log_limit = maxsize * self.CLEANUP_FACTOR + + def _push_entry(self, key): + "Push entry onto our access log, invalidate the old entry if exists." + # Invalidate old entry if it exists + old_entry = self.access_lookup.get(key) + if old_entry: + old_entry.is_valid = False + + new_entry = AccessEntry(key) + + self.access_lookup[key] = new_entry + self.access_log.appendleft(new_entry) + + def _prune_entries(self, num): + "Pop entries from our access log until we popped ``num`` valid ones." + while num > 0: + p = self.access_log.pop() + + if not p.is_valid: + continue # Invalidated entry, skip + + del self._container[p.key] + del self.access_lookup[p.key] + num -= 1 + + def _prune_invalidated_entries(self): + "Rebuild our access_log without the invalidated entries." + self.access_log = deque(e for e in self.access_log if e.is_valid) + + def _get_ordered_access_keys(self): + # Used for testing + return [e.key for e in self.access_log if e.is_valid] + + def __getitem__(self, key): + item = self._container.get(key) + + if not item: + return + + # Insert new entry with new high priority, also implicitly invalidates + # the old entry. + self._push_entry(key) + + if len(self.access_log) > self.access_log_limit: + # Heap is getting too big, try to clean up any tailing invalidated + # entries. + self._prune_invalidated_entries() + + return item + + def __setitem__(self, key, item): + # Add item to our container and access log + self._container[key] = item + self._push_entry(key) + + # Discard invalid and excess entries + self._prune_entries(len(self._container) - self._maxsize) + + + def __delitem__(self, key): + self._invalidate_entry(key) + del self._container[key] + del self._access_lookup[key] + + def __len__(self): + return len(self.access_log) + + def __iter__(self): + return self._container.__iter__() + + def __contains__(self, key): + return self._container.__contains__(key) diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index c4db2378..5756411f 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -1,5 +1,3 @@ -import gzip -import zlib import logging import socket @@ -14,122 +12,25 @@ from socket import error as SocketError, timeout as SocketTimeout try: import ssl BaseSSLError = ssl.SSLError -except ImportError, e: +except ImportError: ssl = None BaseSSLError = None -try: - from cStringIO import StringIO -except ImportError, e: - from StringIO import StringIO - -from filepost import encode_multipart_formdata +from .filepost import encode_multipart_formdata +from .response import HTTPResponse +from .exceptions import ( + SSLError, + MaxRetryError, + TimeoutError, + HostChangedError, + EmptyPoolError) log = logging.getLogger(__name__) -## Exceptions - -class HTTPError(Exception): - "Base exception used by this module." - pass - - -class SSLError(Exception): - "Raised when SSL certificate fails in an HTTPS connection." - pass - - -class MaxRetryError(HTTPError): - "Raised when the maximum number of retries is exceeded." - pass - - -class TimeoutError(HTTPError): - "Raised when a socket timeout occurs." - pass - - -class HostChangedError(HTTPError): - "Raised when an existing pool gets a request for a foreign host." - pass - - -## Response objects - -class HTTPResponse(object): - """ - HTTP Response container. - - Similar to httplib's HTTPResponse but the data is pre-loaded. - """ - - def __init__(self, data='', headers=None, status=0, version=0, reason=None, - strict=0): - self.data = data - self.headers = headers or {} - self.status = status - self.version = version - self.reason = reason - self.strict = strict - - @staticmethod - def from_httplib(r, block=True): - """ - Given an httplib.HTTPResponse instance, return a corresponding - urllib3.HTTPResponse object. - - NOTE: This method will perform r.read() which will have side effects - on the original http.HTTPResponse object. - """ - - if block: - tmp_data = r.read() - try: - if r.getheader('content-encoding') == 'gzip': - log.debug("Received response with content-encoding: gzip, " - "decompressing with gzip.") - - gzipper = gzip.GzipFile(fileobj=StringIO(tmp_data)) - data = gzipper.read() - elif r.getheader('content-encoding') == 'deflate': - log.debug("Received response with content-encoding: deflate, " - "decompressing with zlib.") - try: - data = zlib.decompress(tmp_data) - except zlib.error, e: - data = zlib.decompress(tmp_data, -zlib.MAX_WBITS) - else: - data = tmp_data - - except IOError: - raise HTTPError("Received response with content-encoding: %s, " - "but failed to decompress it." % - (r.getheader('content-encoding'))) - else: - data = None - - resp = HTTPResponse(data=data, - headers=dict(r.getheaders()), - status=r.status, - version=r.version, - reason=r.reason, - strict=r.strict) - - resp._raw = r - return resp - - # Backwards-compatibility methods for httplib.HTTPResponse - def getheaders(self): - return self.headers - - def getheader(self, name, default=None): - return self.headers.get(name, default) - - -## Connection objects +## Connection objects (extension of httplib) class VerifiedHTTPSConnection(HTTPSConnection): """ @@ -137,8 +38,13 @@ class VerifiedHTTPSConnection(HTTPSConnection): SSL certification. """ - def set_cert(self, key_file=None, cert_file=None, cert_reqs='CERT_NONE', - ca_certs=None): + def __init__(self): + HTTPSConnection.__init__() + self.cert_reqs = None + self.ca_certs = None + + def set_cert(self, key_file=None, cert_file=None, + cert_reqs='CERT_NONE', ca_certs=None): ssl_req_scheme = { 'CERT_NONE': ssl.CERT_NONE, 'CERT_OPTIONAL': ssl.CERT_OPTIONAL, @@ -163,7 +69,11 @@ class VerifiedHTTPSConnection(HTTPSConnection): ## Pool objects -class HTTPConnectionPool(object): +class ConnectionPool(object): + pass + + +class HTTPConnectionPool(ConnectionPool): """ Thread-safe connection pool for one host. @@ -215,8 +125,10 @@ class HTTPConnectionPool(object): self.headers = headers or {} # Fill the queue up so that doing get() on it will block properly - [self.pool.put(None) for i in xrange(maxsize)] + for _ in xrange(maxsize): + self.pool.put(None) + # These are mostly for testing and debugging purposes. self.num_connections = 0 self.num_requests = 0 @@ -241,11 +153,13 @@ class HTTPConnectionPool(object): # If this is a persistent connection, check if it got disconnected if conn and conn.sock and select([conn.sock], [], [], 0.0)[0]: # Either data is buffered (bad), or the connection is dropped. - log.warning("Connection pool detected dropped " - "connection, resetting: %s" % self.host) + log.info("Resetting dropped connection: %s" % self.host) conn.close() - except Empty, e: + except Empty: + if self.block: + raise EmptyPoolError("Pool reached maximum size and no more " + "connections are allowed.") pass # Oh well, we'll create a new connection then return conn or self._new_conn() @@ -259,17 +173,37 @@ class HTTPConnectionPool(object): """ try: self.pool.put(conn, block=False) - except Full, e: + except Full: # This should never happen if self.block == True log.warning("HttpConnectionPool is full, discarding connection: %s" % self.host) + def _make_request(self, conn, method, url, **httplib_request_kw): + """ + Perform a request on a given httplib connection object taken from our + pool. + """ + self.num_requests += 1 + + conn.request(method, url, **httplib_request_kw) + conn.sock.settimeout(self.timeout) + httplib_response = conn.getresponse() + + log.debug("\"%s %s %s\" %s %s" % + (method, url, + conn._http_vsn_str, # pylint: disable-msg=W0212 + httplib_response.status, httplib_response.length)) + + return httplib_response + + def is_same_host(self, url): return (url.startswith('/') or get_host(url) == (self.scheme, self.host, self.port)) def urlopen(self, method, url, body=None, headers=None, retries=3, - redirect=True, assert_same_host=True, block=True): + redirect=True, assert_same_host=True, pool_timeout=None, + **response_kw): """ Get a connection from the pool and perform an HTTP request. @@ -298,8 +232,16 @@ class HTTPConnectionPool(object): If True, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When False, you can use the pool on an HTTP proxy and request foreign hosts. + + pool_timeout + If set and the pool is set to block=True, then this method will + block for ``pool_timeout`` seconds and raise EmptyPoolError if no + connection is available within the time period. + + Additional parameters are passed to + ``HTTPResponse.from_httplib(r, **response_kw)`` """ - if headers == None: + if headers is None: headers = self.headers if retries < 0: @@ -316,24 +258,20 @@ class HTTPConnectionPool(object): try: # Request a connection from the queue - conn = self._get_conn() + conn = self._get_conn(timeout=pool_timeout) - # Make the request - self.num_requests += 1 - conn.request(method, url, body=body, headers=headers) - conn.sock.settimeout(self.timeout) - httplib_response = conn.getresponse() - log.debug("\"%s %s %s\" %s %s" % - (method, url, conn._http_vsn_str, - httplib_response.status, httplib_response.length)) + # Make the request on the httplib connection object + httplib_response = self._make_request(conn, method, url, + body=body, headers=headers) - # from_httplib will perform httplib_response.read() which will have - # the side effect of letting us use this connection for another - # request. - response = HTTPResponse.from_httplib(httplib_response, block=block) + # Import httplib's response into our own wrapper object + response = HTTPResponse.from_httplib(httplib_response, + pool=self, + connection=conn, + **response_kw) - # Put the connection back to be reused - self._put_conn(conn) + # The connection will be put back into the pool when + # response.release_conn() is called (implicitly by response.read()) except (SocketTimeout, Empty), e: # Timed out either by socket or queue @@ -363,7 +301,7 @@ class HTTPConnectionPool(object): return response def get_url(self, url, fields=None, headers=None, retries=3, - redirect=True): + redirect=True, **response_kw): """ Wrapper for performing GET with urlopen (see urlopen for more details). @@ -373,10 +311,11 @@ class HTTPConnectionPool(object): if fields: url += '?' + urlencode(fields) return self.urlopen('GET', url, headers=headers, retries=retries, - redirect=redirect) + redirect=redirect, **response_kw) def post_url(self, url, fields=None, headers=None, retries=3, - redirect=True, encode_multipart=True): + redirect=True, encode_multipart=True, multipart_boundary=None, + **response_kw): """ Wrapper for performing POST with urlopen (see urlopen for more details). @@ -404,7 +343,8 @@ class HTTPConnectionPool(object): which is used to compose the body of the request. """ if encode_multipart: - body, content_type = encode_multipart_formdata(fields or {}) + body, content_type = encode_multipart_formdata(fields or {}, + boundary=multipart_boundary) else: body, content_type = ( urlencode(fields or {}), @@ -414,7 +354,7 @@ class HTTPConnectionPool(object): headers.update({'Content-Type': content_type}) return self.urlopen('POST', url, body, headers=headers, - retries=retries, redirect=redirect) + retries=retries, redirect=redirect, **response_kw) class HTTPSConnectionPool(HTTPConnectionPool): @@ -424,28 +364,20 @@ class HTTPSConnectionPool(HTTPConnectionPool): scheme = 'https' - def __init__(self, host, port=None, strict=False, timeout=None, maxsize=1, - block=False, headers=None, key_file=None, - cert_file=None, cert_reqs='CERT_NONE', ca_certs=None): - self.host = host - self.port = port - self.strict = strict - self.timeout = timeout - self.pool = Queue(maxsize) - self.block = block - self.headers = headers or {} + def __init__(self, host, port=None, + strict=False, timeout=None, maxsize=1, + block=False, headers=None, + key_file=None, cert_file=None, + cert_reqs='CERT_NONE', ca_certs=None): + super(HTTPSConnectionPool, self).__init__(host, port, + strict, timeout, maxsize, + block, headers) self.key_file = key_file self.cert_file = cert_file self.cert_reqs = cert_reqs self.ca_certs = ca_certs - # Fill the queue up so that doing get() on it will block properly - [self.pool.put(None) for i in xrange(maxsize)] - - self.num_connections = 0 - self.num_requests = 0 - def _new_conn(self): """ Return a fresh HTTPSConnection. @@ -527,7 +459,7 @@ def get_host(url): if '//' in url: scheme, url = url.split('://', 1) if '/' in url: - url, path = url.split('/', 1) + url, _path = url.split('/', 1) if ':' in url: url, port = url.split(':', 1) port = int(port) diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py new file mode 100644 index 00000000..45b0e822 --- /dev/null +++ b/requests/packages/urllib3/exceptions.py @@ -0,0 +1,29 @@ +## Exceptions + +class HTTPError(Exception): + "Base exception used by this module." + pass + + +class SSLError(Exception): + "Raised when SSL certificate fails in an HTTPS connection." + pass + + +class MaxRetryError(HTTPError): + "Raised when the maximum number of retries is exceeded." + pass + + +class TimeoutError(HTTPError): + "Raised when a socket timeout occurs." + pass + + +class HostChangedError(HTTPError): + "Raised when an existing pool gets a request for a foreign host." + pass + +class EmptyPoolError(HTTPError): + "Raised when a pool runs out of connections and no more are allowed." + pass diff --git a/requests/packages/urllib3/filepost.py b/requests/packages/urllib3/filepost.py index 181822ba..67d2d0d8 100644 --- a/requests/packages/urllib3/filepost.py +++ b/requests/packages/urllib3/filepost.py @@ -1,10 +1,11 @@ import codecs import mimetools import mimetypes + try: from cStringIO import StringIO -except: - from StringIO import StringIO +except ImportError: + from StringIO import StringIO # pylint: disable-msg=W0404 writer = codecs.lookup('utf-8')[3] @@ -14,12 +15,13 @@ def get_content_type(filename): return mimetypes.guess_type(filename)[0] or 'application/octet-stream' -def encode_multipart_formdata(fields): +def encode_multipart_formdata(fields, boundary=None): body = StringIO() - BOUNDARY = mimetools.choose_boundary() + if boundary is None: + boundary = mimetools.choose_boundary() for fieldname, value in fields.iteritems(): - body.write('--%s\r\n' % (BOUNDARY)) + body.write('--%s\r\n' % (boundary)) if isinstance(value, tuple): filename, data = value @@ -43,8 +45,8 @@ def encode_multipart_formdata(fields): body.write('\r\n') - body.write('--%s--\r\n' % (BOUNDARY)) + body.write('--%s--\r\n' % (boundary)) - content_type = 'multipart/form-data; boundary=%s' % BOUNDARY + content_type = 'multipart/form-data; boundary=%s' % boundary return body.getvalue(), content_type diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py new file mode 100644 index 00000000..d451f68a --- /dev/null +++ b/requests/packages/urllib3/poolmanager.py @@ -0,0 +1,64 @@ +from ._collections import RecentlyUsedContainer +from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, get_host + + +pool_classes_by_scheme = { + 'http': HTTPConnectionPool, + 'https': HTTPSConnectionPool, +} + +port_by_scheme = { + 'http': 80, + 'https': 433, +} + + +class PoolManager(object): + """ + Allows for arbitrary requests while transparently keeping track of + necessary connection pools for you. + + num_pools + Number of connection pools to cache before discarding the least recently + used pool. + + Additional parameters are used to create fresh ConnectionPool instances. + + """ + + # TODO: Make sure there are no memory leaks here. + + def __init__(self, num_pools=10, **connection_pool_kw): + self.connection_pool_kw = connection_pool_kw + + self.pools = RecentlyUsedContainer(num_pools) + self.recently_used_pools = [] + + def connection_from_url(self, url): + """ + Similar to connectionpool.connection_from_url but doesn't pass any + additional keywords to the ConnectionPool constructor. Additional + keywords are taken from the PoolManager constructor. + """ + scheme, host, port = get_host(url) + + # If the scheme, host, or port doesn't match existing open connections, + # open a new ConnectionPool. + pool_key = (scheme, host, port or port_by_scheme.get(scheme, 80)) + + pool = self.pools.get(pool_key) + if pool: + return pool + + # Make a fresh ConnectionPool of the desired type + pool_cls = pool_classes_by_scheme[scheme] + pool = pool_cls(host, port, **self.connection_pool_kw) + + self.pools[pool_key] = pool + + return pool + + def urlopen(self, method, url, **kw): + "Same as HTTP(S)ConnectionPool.urlopen" + conn = self.connection_from_url(url) + return conn.urlopen(method, url, **kw) diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py new file mode 100644 index 00000000..3cba3ad4 --- /dev/null +++ b/requests/packages/urllib3/response.py @@ -0,0 +1,173 @@ +import gzip +import logging +import zlib + + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO # pylint: disable-msg=W0404 + + +from .exceptions import HTTPError + + +log = logging.getLogger(__name__) + + +def decode_gzip(data): + gzipper = gzip.GzipFile(fileobj=StringIO(data)) + return gzipper.read() + + +def decode_deflate(data): + try: + return zlib.decompress(data) + except zlib.error: + return zlib.decompress(data, -zlib.MAX_WBITS) + + +class HTTPResponse(object): + """ + HTTP Response container. + + Backwards-compatible to httplib's HTTPResponse but the response ``body`` is + loaded and decoded on-demand when the ``data`` property is accessed. + + Extra parameters for behaviour not present in httplib.HTTPResponse: + + preload_content + If True, the response's body will be preloaded during construction. + + decode_content + If True, attempts to decode specific content-encoding's based on headers + (like 'gzip' and 'deflate') will be skipped and raw data will be used + instead. + + original_response + When this HTTPResponse wrapper is generated from an httplib.HTTPResponse + object, it's convenient to include the original for debug purposes. It's + otherwise unused. + """ + + CONTENT_DECODERS = { + 'gzip': decode_gzip, + 'deflate': decode_deflate, + } + + def __init__(self, body='', headers=None, status=0, version=0, reason=None, + strict=0, preload_content=False, decode_content=True, + original_response=None, pool=None, connection=None): + self.headers = headers or {} + self.status = status + self.version = version + self.reason = reason + self.strict = strict + + self._decode_content = decode_content + self._body = None + self._fp = None + self._original_response = original_response + + self._pool = pool + self._connection = connection + + if hasattr(body, 'read'): + self._fp = body + + if preload_content: + self._body = self.read(decode_content=decode_content) + + def release_conn(self): + if not self._pool or not self._connection: + return + + self._pool._put_conn(self._connection) + + @property + def data(self): + # For backwords-compat with earlier urllib3 0.4 and earlier. + if self._body: + return self._body + + if self._fp: + return self.read(decode_content=self._decode_content, + cache_content=True) + + def read(self, amt=None, decode_content=True, cache_content=False): + """ + Similar to ``httplib.HTTPResponse.read(amt=None)``. + + amt + How much of the content to read. If specified, decoding and caching + is skipped because we can't decode partial content nor does it make + sense to cache partial content as the full response. + + decode_content + If True, will attempt to decode the body based on the + 'content-encoding' header. (Overridden if ``amt`` is set.) + + cache_content + If True, will save the returned data such that the same result is + returned despite of the state of the underlying file object. This + is useful if you want the ``.data`` property to continue working + after having ``.read()`` the file object. (Overridden if ``amt`` is + set.) + """ + content_encoding = self.headers.get('content-encoding') + decoder = self.CONTENT_DECODERS.get(content_encoding) + + data = self._fp and self._fp.read(amt) + + try: + + if amt: + return data + + if not decode_content or not decoder: + if cache_content: + self._body = data + + return data + + try: + data = decoder(data) + except IOError: + raise HTTPError("Received response with content-encoding: %s, but " + "failed to decode it." % content_encoding) + + if cache_content: + self._body = data + + return data + + finally: + + if self._original_response and self._original_response.isclosed(): + self.release_conn() + + @staticmethod + def from_httplib(r, **response_kw): + """ + Given an httplib.HTTPResponse instance ``r``, return a corresponding + urllib3.HTTPResponse object. + + Remaining parameters are passed to the HTTPResponse constructor, along + with ``original_response=r``. + """ + + return HTTPResponse(body=r, + headers=dict(r.getheaders()), + status=r.status, + version=r.version, + reason=r.reason, + strict=r.strict, + original_response=r, + **response_kw) + + # Backwards-compatibility methods for httplib.HTTPResponse + def getheaders(self): + return self.headers + + def getheader(self, name, default=None): + return self.headers.get(name, default) From 3af8af573068de7bb9b34758a1a7eb7ab7eb9e83 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 17:42:02 -0400 Subject: [PATCH 082/255] =?UTF-8?q?remove=20deb=C3=ADan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debian/changelog | 5 ----- debian/compat | 1 - debian/control | 13 ------------- debian/docs | 1 - debian/pyversions | 1 - debian/rules | 42 ------------------------------------------ 6 files changed, 63 deletions(-) delete mode 100644 debian/changelog delete mode 100644 debian/compat delete mode 100644 debian/control delete mode 100644 debian/docs delete mode 100644 debian/pyversions delete mode 100755 debian/rules diff --git a/debian/changelog b/debian/changelog deleted file mode 100644 index 519752c4..00000000 --- a/debian/changelog +++ /dev/null @@ -1,5 +0,0 @@ -python-requests (0.4.1-0) testing; urgency=low - - * Initial Debian package - - -- Bruno Clermont Thu, 26 May 2011 16:25:00 -0500 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index 7ed6ff82..00000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -5 diff --git a/debian/control b/debian/control deleted file mode 100644 index 5c6ce49a..00000000 --- a/debian/control +++ /dev/null @@ -1,13 +0,0 @@ -Source: python-requests -Section: python -Priority: optional -Maintainer: Bruno Clermont -Homepage: https://github.com/bclermont/requests -Bugs: https://github.com/bclermont/requests/issues -Build-Depends: debhelper, python-support - -Package: python-requests -Architecture: all -Depends: ${python:Depends}, python-support -Provides: ${python:Provides} -Description: Python HTTP Requests for Humans. diff --git a/debian/docs b/debian/docs deleted file mode 100644 index 9bb74d2d..00000000 --- a/debian/docs +++ /dev/null @@ -1 +0,0 @@ -docs/user/*.rst \ No newline at end of file diff --git a/debian/pyversions b/debian/pyversions deleted file mode 100644 index 8b253bc3..00000000 --- a/debian/pyversions +++ /dev/null @@ -1 +0,0 @@ -2.4- diff --git a/debian/rules b/debian/rules deleted file mode 100755 index 28e92ec0..00000000 --- a/debian/rules +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/make -f - -# Verbose mode -#export DH_VERBOSE=1 - -clean: - dh_testdir - dh_testroot - - rm -rf build requests.egg-info -# find django-sentry/ -name *.pyc | xargs rm -f - - dh_clean - -build: - dh_testdir - - python setup.py build - -install: - dh_testdir - dh_installdirs - - python setup.py install --root $(CURDIR)/debian/python-requests - -binary-indep: install - -binary-arch: install - dh_install - dh_installdocs -# dh_installchangelogs - dh_compress - dh_fixperms - dh_pysupport - dh_gencontrol - dh_installdeb - dh_md5sums - dh_builddeb -- -Z lzma -z9 - -binary: binary-indep binary-arch -.PHONY: build clean binary-indep binary-arch binary - From 37ed9116b7081ff2b1542af7bdb2857f3ca63661 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 17:42:13 -0400 Subject: [PATCH 083/255] remove async (for now) --- requests/async.py | 41 ----------------------------------------- 1 file changed, 41 deletions(-) delete mode 100644 requests/async.py diff --git a/requests/async.py b/requests/async.py deleted file mode 100644 index ab04084f..00000000 --- a/requests/async.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - requests.async - ~~~~~~~~~~~~~~ - - This module implements the main Requests system, after monkey-patching - the urllib2 module with eventlet or gevent.. - - :copyright: (c) 2011 by Kenneth Reitz. - :license: ISC, see LICENSE for more details. -""" - - -from __future__ import absolute_import - -import urllib -import urllib2 - -from urllib2 import HTTPError - - -try: - import eventlet - eventlet.monkey_patch() -except ImportError: - pass - -if not 'eventlet' in locals(): - try: - from gevent import monkey - monkey.patch_all() - except ImportError: - pass - - -if not 'eventlet' in locals(): - raise ImportError('No Async adaptations of urllib2 found!') - - -from .core import * From 33df9d1bf2cf24bf46af0d079a3f4884816dd9e6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 17:42:19 -0400 Subject: [PATCH 084/255] monkeys are deprecated --- requests/monkeys.py | 142 -------------------------------------------- 1 file changed, 142 deletions(-) delete mode 100644 requests/monkeys.py diff --git a/requests/monkeys.py b/requests/monkeys.py deleted file mode 100644 index 8f10d29f..00000000 --- a/requests/monkeys.py +++ /dev/null @@ -1,142 +0,0 @@ -#-*- coding: utf-8 -*- - -""" -requests.monkeys -~~~~~~~~~~~~~~~~ - -Urllib2 Monkey patches. - -""" - -import re -import urllib2 - - -class Request(urllib2.Request): - """Hidden wrapper around the urllib2.Request object. Allows for manual - setting of HTTP methods. - """ - - def __init__(self, url, data=None, headers={}, origin_req_host=None, unverifiable=False, method=None): - urllib2.Request.__init__(self, url, data, headers, origin_req_host, unverifiable) - self.method = method - - def get_method(self): - if self.method: - return self.method - - return urllib2.Request.get_method(self) - - -class HTTPRedirectHandler(urllib2.HTTPRedirectHandler): - """HTTP Redirect handler.""" - def http_error_301(self, req, fp, code, msg, headers): - pass - - http_error_302 = http_error_303 = http_error_307 = http_error_301 - - -class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler): - """HTTP Basic Auth Handler with authentication loop fixes.""" - - def __init__(self, *args, **kwargs): - urllib2.HTTPBasicAuthHandler.__init__(self, *args, **kwargs) - self.retried_req = None - self.retried = 0 - - def reset_retry_count(self): - # Python 2.6.5 will call this on 401 or 407 errors and thus loop - # forever. We disable reset_retry_count completely and reset in - # http_error_auth_reqed instead. - pass - - def http_error_auth_reqed(self, auth_header, host, req, headers): - # Reset the retry counter once for each request. - if req is not self.retried_req: - self.retried_req = req - self.retried = 0 - - return urllib2.HTTPBasicAuthHandler.http_error_auth_reqed( - self, auth_header, host, req, headers - ) - - -class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler): - """HTTP Basic Auth Handler with forced Authentication.""" - - auth_header = 'Authorization' - rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+' - 'realm=(["\'])(.*?)\\2', re.I) - - def __init__(self, *args, **kwargs): - HTTPBasicAuthHandler.__init__(self, *args, **kwargs) - - def http_error_401(self, req, fp, code, msg, headers): - url = req.get_full_url() - response = self._http_error_auth_reqed('www-authenticate', url, req, headers) - self.reset_retry_count() - return response - - http_error_404 = http_error_401 - - def _http_error_auth_reqed(self, authreq, host, req, headers): - - authreq = headers.get(authreq, None) - - if self.retried > 5: - # retry sending the username:password 5 times before failing. - raise urllib2.HTTPError(req.get_full_url(), 401, "basic auth failed", - headers, None) - else: - self.retried += 1 - - if authreq: - - mo = self.rx.search(authreq) - - if mo: - scheme, quote, realm = mo.groups() - - if scheme.lower() == 'basic': - response = self.retry_http_basic_auth(host, req, realm) - - if response and response.code not in (401, 404): - self.retried = 0 - return response - else: - response = self.retry_http_basic_auth(host, req, 'Realm') - - if response and response.code not in (401, 404): - self.retried = 0 - return response - - -class HTTPDigestAuthHandler(urllib2.HTTPDigestAuthHandler): - - def __init__(self, *args, **kwargs): - urllib2.HTTPDigestAuthHandler.__init__(self, *args, **kwargs) - self.retried_req = None - - def reset_retry_count(self): - # Python 2.6.5 will call this on 401 or 407 errors and thus loop - # forever. We disable reset_retry_count completely and reset in - # http_error_auth_reqed instead. - pass - - def http_error_auth_reqed(self, auth_header, host, req, headers): - # Reset the retry counter once for each request. - if req is not self.retried_req: - self.retried_req = req - self.retried = 0 - # In python < 2.5 AbstractDigestAuthHandler raises a ValueError if - # it doesn't know about the auth type requested. This can happen if - # somebody is using BasicAuth and types a bad password. - - try: - return urllib2.HTTPDigestAuthHandler.http_error_auth_reqed( - self, auth_header, host, req, headers) - except ValueError, inst: - arg = inst.args[0] - if arg.startswith("AbstractDigestAuthHandler doesn't know "): - return - raise From 55cea74b5643e660cc5cee9a4b0abe4cfb3a2f27 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 17:42:27 -0400 Subject: [PATCH 085/255] urllib3 update --- requests/packages/urllib3/connectionpool.py | 29 +++++++++++++++++---- requests/packages/urllib3/response.py | 2 +- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 5756411f..20770260 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -203,7 +203,7 @@ class HTTPConnectionPool(ConnectionPool): def urlopen(self, method, url, body=None, headers=None, retries=3, redirect=True, assert_same_host=True, pool_timeout=None, - **response_kw): + release_conn=None, **response_kw): """ Get a connection from the pool and perform an HTTP request. @@ -238,6 +238,14 @@ class HTTPConnectionPool(ConnectionPool): block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. + release_conn + If False, then the urlopen call will not release the connection + back into the pool once a response is received. This is useful if + you're not preloading the response's content immediately. You will + need to call ``r.release_conn()`` on the response ``r`` to return + the connection back into the pool. If None, it takes the value of + ``response_kw.get('preload_content', True)``. + Additional parameters are passed to ``HTTPResponse.from_httplib(r, **response_kw)`` """ @@ -247,6 +255,9 @@ class HTTPConnectionPool(ConnectionPool): if retries < 0: raise MaxRetryError("Max retries exceeded for url: %s" % url) + if release_conn is None: + release_conn = response_kw.get('preload_content', True) + # Check host if assert_same_host and not self.is_same_host(url): host = "%s://%s" % (self.scheme, self.host) @@ -256,14 +267,13 @@ class HTTPConnectionPool(ConnectionPool): raise HostChangedError("Connection pool with host '%s' tried to " "open a foreign host: %s" % (host, url)) - try: - # Request a connection from the queue - conn = self._get_conn(timeout=pool_timeout) + # Request a connection from the queue + conn = self._get_conn(timeout=pool_timeout) + try: # Make the request on the httplib connection object httplib_response = self._make_request(conn, method, url, body=body, headers=headers) - # Import httplib's response into our own wrapper object response = HTTPResponse.from_httplib(httplib_response, pool=self, @@ -283,6 +293,15 @@ class HTTPConnectionPool(ConnectionPool): raise SSLError(e) except (HTTPException, SocketError), e: + # Connection broken, discard. It will be replaced next _get_conn(). + conn = None + + finally: + if release_conn: + # Put the connection back to be reused + self._put_conn(conn) + + if not conn: log.warn("Retrying (%d attempts remain) after connection " "broken by '%r': %s" % (retries, e, url)) return self.urlopen(method, url, body, headers, retries - 1, diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 3cba3ad4..14b30682 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -56,7 +56,7 @@ class HTTPResponse(object): } def __init__(self, body='', headers=None, status=0, version=0, reason=None, - strict=0, preload_content=False, decode_content=True, + strict=0, preload_content=True, decode_content=True, original_response=None, pool=None, connection=None): self.headers = headers or {} self.status = status From 182f92ab2602658802724d1204feade7a7117994 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 18:00:37 -0400 Subject: [PATCH 086/255] hrmm --- requests/core.py | 2 +- requests/models.py | 194 ++++++++++++++++----------------------------- requests/utils.py | 65 +++++++++++++++ 3 files changed, 134 insertions(+), 127 deletions(-) diff --git a/requests/core.py b/requests/core.py index c9f10f55..477f39d7 100644 --- a/requests/core.py +++ b/requests/core.py @@ -22,7 +22,7 @@ __copyright__ = 'Copyright 2011 Kenneth Reitz' from api import * from config import settings from exceptions import * -from models import HTTPError, Request, Response +from models import Request, Response from sessions import session from status_codes import codes diff --git a/requests/models.py b/requests/models.py index a90924db..4ff80ca1 100644 --- a/requests/models.py +++ b/requests/models.py @@ -11,8 +11,6 @@ import urllib2 import socket import zlib - -from urllib2 import HTTPError from urlparse import urlparse, urlunparse, urljoin from datetime import datetime @@ -22,7 +20,7 @@ from .packages import urllib3 from .config import settings from .structures import CaseInsensitiveDict -from .utils import dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip +from .utils import * from .status_codes import codes from .exceptions import RequestException, Timeout, URLRequired, TooManyRedirects @@ -74,8 +72,10 @@ class Request(object): # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) self.proxies = proxies - self.data, self._enc_data = self._encode_params(data) - self.params, self._enc_params = self._encode_params(params) + self.data = data + self._enc_data = encode_params(data) + self.params = params + self._enc_params = encode_params(params) #: :class:`Response ` instance, containing #: content and metadata of HTTP Response, once :attr:`sent `. @@ -277,66 +277,7 @@ class Request(object): # Give Response some context. self.response.request = self - @staticmethod - def _encode_params(data): - """Encode parameters in a piece of data. - If the data supplied is a dictionary, encodes each parameter in it, and - returns a list of tuples containing the encoded parameters, and a urlencoded - version of that. - - Otherwise, assumes the data is already encoded appropriately, and - returns it twice. - """ - - if hasattr(data, 'items'): - result = [] - for k, vs in data.items(): - for v in isinstance(vs, list) and vs or [vs]: - result.append((k.encode('utf-8') if isinstance(k, unicode) else k, - v.encode('utf-8') if isinstance(v, unicode) else v) - ) - return (result, urllib.urlencode(result, doseq=True)) - - else: - return data, data - - def _build_url(self): - """Build the actual URL to use.""" - - # Support for unicode domain names and paths. - (scheme, netloc, path, params, query, fragment) = urlparse(self.url) - - # International Domain Name - netloc = netloc.encode('idna') - - # Encode the path to to utf-8. - if isinstance(path, unicode): - path = path.encode('utf-8') - - # URL-encode the path. - path = urllib.quote(path, safe="%/:=&?~#+!$,;'@()*[]") - - # Turn it back into a bytestring. - self.url = str(urlunparse([scheme, netloc, path, params, query, fragment])) - self.url = str(urlunparse( - [scheme, netloc, path, params, query, fragment] - )) - - # Query Parameters? - if self._enc_params: - - # If query parameters already exist in the URL, append. - if urlparse(self.url).query: - return '%s&%s' % (self.url, self._enc_params) - - # Otherwise, have at it. - else: - return '%s?%s' % (self.url, self._enc_params) - - else: - # Kosher URL. - return self.url def send(self, connection=None, anyway=False): """Sends the shit.""" @@ -345,8 +286,9 @@ class Request(object): self._checks() # Build the final URL. - url = self._build_url() + url = build_url(self.url, self.params) + print url # Setup Files. if self.files: pass @@ -367,8 +309,7 @@ class Request(object): try: # Create a new HTTP connection, since one wasn't passed in. if not connection: - connection = urllib3.connection_from_url(url, - timeout=self.timeout) + connection = urllib3.connection_from_url(url, timeout=self.timeout) # One-off request. Delay fetching the content until needed. do_block = False @@ -384,7 +325,8 @@ class Request(object): headers=self.headers, redirect=False, assert_same_host=False, - block=do_block + preload_content=do_block + # block=do_block ) # Extract cookies. @@ -412,87 +354,87 @@ class Request(object): - def old_send(self, anyway=False): - """Sends the request. Returns True of successful, false if not. - If there was an HTTPError during transmission, - self.response.status_code will contain the HTTPError code. + # def old_send(self, anyway=False): + # """Sends the request. Returns True of successful, false if not. + # If there was an HTTPError during transmission, + # self.response.status_code will contain the HTTPError code. - Once a request is successfully sent, `sent` will equal True. + # Once a request is successfully sent, `sent` will equal True. - :param anyway: If True, request will be sent, even if it has - already been sent. - """ + # :param anyway: If True, request will be sent, even if it has + # already been sent. + # """ - self._checks() + # self._checks() - # Logging - if settings.verbose: - settings.verbose.write('%s %s %s\n' % ( - datetime.now().isoformat(), self.method, self.url - )) + # # Logging + # if settings.verbose: + # settings.verbose.write('%s %s %s\n' % ( + # datetime.now().isoformat(), self.method, self.url + # )) - url = self._build_url() - if self.method in ('GET', 'HEAD', 'DELETE'): - req = _Request(url, method=self.method) - else: + # url = self._build_url() + # if self.method in ('GET', 'HEAD', 'DELETE'): + # req = _Request(url, method=self.method) + # else: - if self.files: - register_openers() + # if self.files: + # register_openers() - if self.data: - self.files.update(self.data) + # if self.data: + # self.files.update(self.data) - datagen, headers = multipart_encode(self.files) - req = _Request(url, data=datagen, headers=headers, method=self.method) + # datagen, headers = multipart_encode(self.files) + # req = _Request(url, data=datagen, headers=headers, method=self.method) - else: - req = _Request(url, data=self._enc_data, method=self.method) + # else: + # req = _Request(url, data=self._enc_data, method=self.method) - if self.headers: - for k, v in self.headers.iteritems(): - req.add_header(k, v) + # if self.headers: + # for k, v in self.headers.iteritems(): + # req.add_header(k, v) - if not self.sent or anyway: + # if not self.sent or anyway: - try: - opener = self._get_opener() - try: + # try: + # opener = self._get_opener() + # try: - resp = opener(req, timeout=self.timeout) + # resp = opener(req, timeout=self.timeout) - except TypeError, err: - # timeout argument is new since Python v2.6 - if not 'timeout' in str(err): - raise + # except TypeError, err: + # # timeout argument is new since Python v2.6 + # if not 'timeout' in str(err): + # raise - if settings.timeout_fallback: - # fall-back and use global socket timeout (This is not thread-safe!) - old_timeout = socket.getdefaulttimeout() - socket.setdefaulttimeout(self.timeout) + # if settings.timeout_fallback: + # # fall-back and use global socket timeout (This is not thread-safe!) + # old_timeout = socket.getdefaulttimeout() + # socket.setdefaulttimeout(self.timeout) - resp = opener(req) + # resp = opener(req) - if settings.timeout_fallback: - # restore gobal timeout - socket.setdefaulttimeout(old_timeout) + # if settings.timeout_fallback: + # # restore gobal timeout + # socket.setdefaulttimeout(old_timeout) - if self.cookiejar is not None: - self.cookiejar.extract_cookies(resp, req) + # if self.cookiejar is not None: + # self.cookiejar.extract_cookies(resp, req) - except (urllib2.HTTPError, urllib2.URLError), why: - if hasattr(why, 'reason'): - if isinstance(why.reason, socket.timeout): - why = Timeout(why) + # except (urllib2.HTTPError, urllib2.URLError), why: + # if hasattr(why, 'reason'): + # if isinstance(why.reason, socket.timeout): + # why = Timeout(why) - self._build_response(why, is_error=True) + # self._build_response(why, is_error=True) - else: - self._build_response(resp) - self.response.ok = True + # else: + # self._build_response(resp) + # self.response.ok = True - self.sent = self.response.ok + # self.sent = self.response.ok - return self.sent + # return self.sent class Response(object): diff --git a/requests/utils.py b/requests/utils.py index 75357ae7..62941fb3 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -13,7 +13,72 @@ import cgi import codecs import cookielib import re +import urllib import zlib +from urlparse import urlparse, urlunparse, urljoin + + +def encode_params(params): + """Encode parameters in a piece of data. + + If the data supplied is a dictionary, encodes each parameter in it, and + returns a list of tuples containing the encoded parameters, and a urlencoded + version of that. + + Otherwise, assumes the data is already encoded appropriately, and + returns it twice. + """ + + if hasattr(params, 'items'): + result = [] + for k, vs in params.items(): + for v in isinstance(vs, list) and vs or [vs]: + result.append((k.encode('utf-8') if isinstance(k, unicode) else k, + v.encode('utf-8') if isinstance(v, unicode) else v) + ) + return urllib.urlencode(result, doseq=True) + + else: + return params + +def build_url(url, params): + """Build the actual URL to use.""" + + + # Support for unicode domain names and paths. + (scheme, netloc, path, params, query, fragment) = urlparse(url) + + # International Domain Name + netloc = netloc.encode('idna') + + # Encode the path to to utf-8. + if isinstance(path, unicode): + path = path.encode('utf-8') + + # URL-encode the path. + path = urllib.quote(path, safe="%/:=&?~#+!$,;'@()*[]") + + # Turn it back into a bytestring. + url = str(urlunparse([scheme, netloc, path, params, query, fragment])) + url = str(urlunparse( + [scheme, netloc, path, params, query, fragment] + )) + + # Query Parameters? + if params: + params = encode_params(params) + + # If query parameters already exist in the URL, append. + if urlparse(url).query: + return '%s&%s' % (url, params) + + # Otherwise, have at it. + else: + return '%s?%s' % (url, params) + + else: + # Kosher URL. + return url def header_expand(headers): From 0b22bc9b4243808628e417959a4ada0d803ae515 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 18:02:22 -0400 Subject: [PATCH 087/255] cookie jar => cookies --- requests/api.py | 2 +- requests/models.py | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/requests/api.py b/requests/api.py index fcd02b56..0f380a60 100644 --- a/requests/api.py +++ b/requests/api.py @@ -61,7 +61,7 @@ def request(method, url, data=data, params=params, headers=headers, - cookiejar=cookies, + cookies=cookies, files=files, auth=auth, timeout=timeout or config.settings.timeout, diff --git a/requests/models.py b/requests/models.py index 4ff80ca1..d7d32374 100644 --- a/requests/models.py +++ b/requests/models.py @@ -35,7 +35,7 @@ class Request(object): def __init__(self, url=None, headers=dict(), files=None, method=None, data=dict(), - params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False, + params=dict(), auth=None, cookies=None, timeout=None, redirect=False, allow_redirects=False, proxies=None): #: Float describ the timeout of the request. @@ -90,7 +90,7 @@ class Request(object): self.auth = auth #: CookieJar to attach to :class:`Request `. - self.cookiejar = cookiejar + self.cookies = cookies #: True if Request has been sent. self.sent = False @@ -298,10 +298,8 @@ class Request(object): pass # Setup cookies. - # elif self.cookies: - # pass - - # req = _Request(url, data=self._enc_data, method=self.method) + elif self.cookies: + pass # Only send the Request if new or forced. if (anyway) or (not self.sent): From 5287b8cd9f1eca624b57cb03ad5d1fb2db41bf9e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 18:57:17 -0400 Subject: [PATCH 088/255] fix --- requests/models.py | 261 ++++++--------------------------------------- requests/utils.py | 39 +++---- 2 files changed, 48 insertions(+), 252 deletions(-) diff --git a/requests/models.py b/requests/models.py index d7d32374..2072621c 100644 --- a/requests/models.py +++ b/requests/models.py @@ -81,13 +81,13 @@ class Request(object): #: content and metadata of HTTP Response, once :attr:`sent `. self.response = Response() - if isinstance(auth, (list, tuple)): - auth = AuthObject(*auth) - if not auth: - auth = auth_manager.get_auth(self.url) + # if isinstance(auth, (list, tuple)): + # auth = AuthObject(*auth) + # if not auth: + # auth = auth_manager.get_auth(self.url) #: :class:`AuthObject` to attach to :class:`Request `. - self.auth = auth + # self.auth = auth #: CookieJar to attach to :class:`Request `. self.cookies = cookies @@ -180,7 +180,15 @@ class Request(object): try: response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) - response.raw = resp._raw + response.raw = resp + # print dir(self.response.raw) + # print self + + # print (response.raw) + # print dir(response.raw) + # print response.raw.sock.read() + # print '------' + # if self.cookiejar: @@ -323,7 +331,8 @@ class Request(object): headers=self.headers, redirect=False, assert_same_host=False, - preload_content=do_block + # preload_content=True + preload_content=False # block=do_block ) @@ -342,6 +351,7 @@ class Request(object): print why else: + # self.response = Response.from_urllib3() self._build_response(r) self.response.ok = True @@ -351,90 +361,6 @@ class Request(object): return self.sent - - # def old_send(self, anyway=False): - # """Sends the request. Returns True of successful, false if not. - # If there was an HTTPError during transmission, - # self.response.status_code will contain the HTTPError code. - - # Once a request is successfully sent, `sent` will equal True. - - # :param anyway: If True, request will be sent, even if it has - # already been sent. - # """ - - # self._checks() - - # # Logging - # if settings.verbose: - # settings.verbose.write('%s %s %s\n' % ( - # datetime.now().isoformat(), self.method, self.url - # )) - - # url = self._build_url() - # if self.method in ('GET', 'HEAD', 'DELETE'): - # req = _Request(url, method=self.method) - # else: - - # if self.files: - # register_openers() - - # if self.data: - # self.files.update(self.data) - - # datagen, headers = multipart_encode(self.files) - # req = _Request(url, data=datagen, headers=headers, method=self.method) - - # else: - # req = _Request(url, data=self._enc_data, method=self.method) - - # if self.headers: - # for k, v in self.headers.iteritems(): - # req.add_header(k, v) - - # if not self.sent or anyway: - - # try: - # opener = self._get_opener() - # try: - - # resp = opener(req, timeout=self.timeout) - - # except TypeError, err: - # # timeout argument is new since Python v2.6 - # if not 'timeout' in str(err): - # raise - - # if settings.timeout_fallback: - # # fall-back and use global socket timeout (This is not thread-safe!) - # old_timeout = socket.getdefaulttimeout() - # socket.setdefaulttimeout(self.timeout) - - # resp = opener(req) - - # if settings.timeout_fallback: - # # restore gobal timeout - # socket.setdefaulttimeout(old_timeout) - - # if self.cookiejar is not None: - # self.cookiejar.extract_cookies(resp, req) - - # except (urllib2.HTTPError, urllib2.URLError), why: - # if hasattr(why, 'reason'): - # if isinstance(why.reason, socket.timeout): - # why = Timeout(why) - - # self._build_response(why, is_error=True) - - # else: - # self._build_response(resp) - # self.response.ok = True - - # self.sent = self.response.ok - - # return self.sent - - class Response(object): """The core :class:`Response ` object. @@ -501,13 +427,17 @@ class Response(object): break yield chunk self._content_consumed = True - gen = generate() + gen = generate + if 'gzip' in self.headers.get('content-encoding', ''): gen = stream_decode_gzip(gen) + if decode_unicode is None: decode_unicode = settings.decode_unicode + if decode_unicode: gen = stream_decode_response_unicode(gen, self) + return gen @@ -526,7 +456,18 @@ class Response(object): # Read the contents. # print self.raw.__dict__ - self._content = self.raw.read() or self._response.data + + # print + # print '~' + # # print self.raw + # print self.raw.getresponse() + # print dir(self.raw) + # print + # print + # print + + # self.raw.read() or + self._content = self.raw.read() # print self.raw.__dict__ # Decode GZip'd content. @@ -543,6 +484,7 @@ class Response(object): self._content_consumed = True return self._content + def raise_for_status(self): """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured. @@ -562,139 +504,6 @@ class Response(object): raise Exception('500 yo') -class AuthManager(object): - """Requests Authentication Manager.""" - - def __new__(cls): - singleton = cls.__dict__.get('__singleton__') - if singleton is not None: - return singleton - - cls.__singleton__ = singleton = object.__new__(cls) - - return singleton - - def __init__(self): - self.passwd = {} - self._auth = {} - - def __repr__(self): - return '' % (self.method) - - def add_auth(self, uri, auth): - """Registers AuthObject to AuthManager.""" - - uri = self.reduce_uri(uri, False) - - # try to make it an AuthObject - if not isinstance(auth, AuthObject): - try: - auth = AuthObject(*auth) - except TypeError: - pass - - self._auth[uri] = auth - - def add_password(self, realm, uri, user, passwd): - """Adds password to AuthManager.""" - # uri could be a single URI or a sequence - if isinstance(uri, basestring): - uri = [uri] - - reduced_uri = tuple([self.reduce_uri(u, False) for u in uri]) - - if reduced_uri not in self.passwd: - self.passwd[reduced_uri] = {} - self.passwd[reduced_uri] = (user, passwd) - - def find_user_password(self, realm, authuri): - for uris, authinfo in self.passwd.iteritems(): - reduced_authuri = self.reduce_uri(authuri, False) - for uri in uris: - if self.is_suburi(uri, reduced_authuri): - return authinfo - - return (None, None) - - def get_auth(self, uri): - (in_domain, in_path) = self.reduce_uri(uri, False) - - for domain, path, authority in ( - (i[0][0], i[0][1], i[1]) for i in self._auth.iteritems() - ): - if in_domain == domain: - if path in in_path: - return authority - - def reduce_uri(self, uri, default_port=True): - """Accept authority or URI and extract only the authority and path.""" - - # note HTTP URLs do not have a userinfo component - parts = urllib2.urlparse.urlsplit(uri) - - if parts[1]: - # URI - scheme = parts[0] - authority = parts[1] - path = parts[2] or '/' - else: - # host or host:port - scheme = None - authority = uri - path = '/' - - host, port = urllib2.splitport(authority) - - if default_port and port is None and scheme is not None: - dport = {"http": 80, - "https": 443, - }.get(scheme) - if dport is not None: - authority = "%s:%d" % (host, dport) - - return authority, path - - def is_suburi(self, base, test): - """Check if test is below base in a URI tree - - Both args must be URIs in reduced form. - """ - if base == test: - return True - if base[0] != test[0]: - return False - common = urllib2.posixpath.commonprefix((base[1], test[1])) - if len(common) == len(base[1]): - return True - return False - - def empty(self): - self.passwd = {} - - def remove(self, uri, realm=None): - # uri could be a single URI or a sequence - if isinstance(uri, basestring): - uri = [uri] - - for default_port in True, False: - reduced_uri = tuple([self.reduce_uri(u, default_port) for u in uri]) - del self.passwd[reduced_uri][realm] - - def __contains__(self, uri): - # uri could be a single URI or a sequence - if isinstance(uri, basestring): - uri = [uri] - - uri = tuple([self.reduce_uri(u, False) for u in uri]) - - if uri in self.passwd: - return True - - return False - -auth_manager = AuthManager() - - class AuthObject(object): """The :class:`AuthObject` is a simple HTTP Authentication token. diff --git a/requests/utils.py b/requests/utils.py index 62941fb3..9560f1e8 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -41,44 +41,32 @@ def encode_params(params): else: return params -def build_url(url, params): + +def build_url(url, query_params): """Build the actual URL to use.""" - # Support for unicode domain names and paths. - (scheme, netloc, path, params, query, fragment) = urlparse(url) - - # International Domain Name + scheme, netloc, path, params, query, fragment = urlparse(url) netloc = netloc.encode('idna') - # Encode the path to to utf-8. if isinstance(path, unicode): - path = path.encode('utf-8') + path = path.encode('utf-8') - # URL-encode the path. path = urllib.quote(path, safe="%/:=&?~#+!$,;'@()*[]") - # Turn it back into a bytestring. - url = str(urlunparse([scheme, netloc, path, params, query, fragment])) url = str(urlunparse( - [scheme, netloc, path, params, query, fragment] - )) + [scheme, netloc, path, params, query, fragment] + )) - # Query Parameters? - if params: - params = encode_params(params) - - # If query parameters already exist in the URL, append. - if urlparse(url).query: - return '%s&%s' % (url, params) - - # Otherwise, have at it. - else: - return '%s?%s' % (url, params) + query_params = encode_params(query_params) + if query_params: + if urlparse(url).query: + return '%s&%s' % (url, query_params) + else: + return '%s?%s' % (url, query_params) else: - # Kosher URL. - return url + return url def header_expand(headers): @@ -137,7 +125,6 @@ def dict_from_cookiejar(cj): for _, cookies in cj._cookies.items(): for _, cookies in cookies.items(): for cookie in cookies.values(): - # print cookie cookie_dict[cookie.name] = cookie.value return cookie_dict From 8565053c9355f3760403d41038cc73c5662e2ba9 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 19:01:40 -0400 Subject: [PATCH 089/255] works for block and non-block connections --- requests/models.py | 8 ++++---- requests/utils.py | 9 ++++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/requests/models.py b/requests/models.py index 2072621c..a8ec0de4 100644 --- a/requests/models.py +++ b/requests/models.py @@ -332,8 +332,8 @@ class Request(object): redirect=False, assert_same_host=False, # preload_content=True - preload_content=False - # block=do_block + # preload_content=False + preload_content=do_block ) # Extract cookies. @@ -466,8 +466,8 @@ class Response(object): # print # print - # self.raw.read() or - self._content = self.raw.read() + # self.raw.read() or + self._content = self.raw.read() or self.raw.data # print self.raw.__dict__ # Decode GZip'd content. diff --git a/requests/utils.py b/requests/utils.py index 9560f1e8..30ab9016 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -15,7 +15,7 @@ import cookielib import re import urllib import zlib -from urlparse import urlparse, urlunparse, urljoin +from urlparse import urlparse, urlunparse def encode_params(params): @@ -33,8 +33,11 @@ def encode_params(params): result = [] for k, vs in params.items(): for v in isinstance(vs, list) and vs or [vs]: - result.append((k.encode('utf-8') if isinstance(k, unicode) else k, - v.encode('utf-8') if isinstance(v, unicode) else v) + result.append( + ( + k.encode('utf-8') if isinstance(k, unicode) else k, + v.encode('utf-8') if isinstance(v, unicode) else v + ) ) return urllib.urlencode(result, doseq=True) From bd2b0c45baba6ab6b4fabcf5d7588a63db1865bd Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 19:39:21 -0400 Subject: [PATCH 090/255] simplify --- requests/models.py | 115 --------------------------------------------- 1 file changed, 115 deletions(-) diff --git a/requests/models.py b/requests/models.py index a8ec0de4..d89f2fd2 100644 --- a/requests/models.py +++ b/requests/models.py @@ -81,11 +81,6 @@ class Request(object): #: content and metadata of HTTP Response, once :attr:`sent `. self.response = Response() - # if isinstance(auth, (list, tuple)): - # auth = AuthObject(*auth) - # if not auth: - # auth = auth_manager.get_auth(self.url) - #: :class:`AuthObject` to attach to :class:`Request `. # self.auth = auth @@ -120,54 +115,6 @@ class Request(object): if not self.url: raise URLRequired - def _get_opener(self): - """Creates appropriate opener object for urllib2.""" - - _handlers = [] - - if self.cookiejar is not None: - _handlers.append(urllib2.HTTPCookieProcessor(self.cookiejar)) - - if self.auth: - if not isinstance(self.auth.handler, - (urllib2.AbstractBasicAuthHandler, - urllib2.AbstractDigestAuthHandler)): - - # TODO: REMOVE THIS COMPLETELY - auth_manager.add_password( - self.auth.realm, self.url, - self.auth.username, - self.auth.password) - - self.auth.handler = self.auth.handler(auth_manager) - auth_manager.add_auth(self.url, self.auth) - - _handlers.append(self.auth.handler) - - if self.proxies: - _handlers.append(urllib2.ProxyHandler(self.proxies)) - - _handlers.append(HTTPRedirectHandler) - - if not _handlers: - return urllib2.urlopen - - if self.data or self.files: - _handlers.extend(get_handlers()) - - opener = urllib2.build_opener(*_handlers) - - if self.headers: - # Allow default headers in the opener to be overloaded - normal_keys = [k.capitalize() for k in self.headers] - for key, val in opener.addheaders[:]: - if key not in normal_keys: - continue - # Remove it, we have a value to take its place - opener.addheaders.remove((key, val)) - - return opener.open - def _build_response(self, resp, is_error=False): """Build internal :class:`Response ` object from given response. @@ -181,23 +128,6 @@ class Request(object): try: response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) response.raw = resp - # print dir(self.response.raw) - # print self - - # print (response.raw) - # print dir(response.raw) - # print response.raw.sock.read() - # print '------' - - - # if self.cookiejar: - - # response.cookies = dict_from_cookiejar(self.cookiejar) - - - # response.cookies = dict_from_cookiejar(self.cookiejar) - - # response.cookies = dict_from_cookiejar(self.cookiejar) except AttributeError: pass @@ -455,20 +385,7 @@ class Response(object): 'The content for this response was already consumed') # Read the contents. - # print self.raw.__dict__ - - # print - # print '~' - # # print self.raw - # print self.raw.getresponse() - # print dir(self.raw) - # print - # print - # print - - # self.raw.read() or self._content = self.raw.read() or self.raw.data - # print self.raw.__dict__ # Decode GZip'd content. if 'gzip' in self.headers.get('content-encoding', ''): @@ -502,35 +419,3 @@ class Response(object): elif (self.status_code >= 500) and (self.status_code < 600): raise Exception('500 yo') - - -class AuthObject(object): - """The :class:`AuthObject` is a simple HTTP Authentication token. - - When given to a Requests function, it enables Basic HTTP Authentication - for that Request. You can also enable Authorization for domain realms - with AutoAuth. See AutoAuth for more details. - - :param username: Username to authenticate with. - :param password: Password for given username. - :param realm: (optional) the realm this auth applies to - :param handler: (optional) basic || digest || proxy_basic || proxy_digest - """ - - _handlers = { - # 'basic': HTTPBasicAuthHandler, - # 'forced_basic': HTTPForcedBasicAuthHandler, - # 'digest': HTTPDigestAuthHandler, - # 'proxy_basic': urllib2.ProxyBasicAuthHandler, - # 'proxy_digest': urllib2.ProxyDigestAuthHandler - } - - def __init__(self, username, password, handler='forced_basic', realm=None): - self.username = username - self.password = password - self.realm = realm - - if isinstance(handler, basestring): - self.handler = self._handlers.get(handler.lower(), HTTPForcedBasicAuthHandler) - else: - self.handler = handler From 9488af5a42bb02913d99e2255cd53f2ad7c860e9 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 19:41:48 -0400 Subject: [PATCH 091/255] cleanups --- requests/api.py | 2 +- requests/models.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requests/api.py b/requests/api.py index 0f380a60..1e4bd688 100644 --- a/requests/api.py +++ b/requests/api.py @@ -12,7 +12,7 @@ This module impliments the Requests API. """ import config -from .models import Request, Response, AuthObject +from .models import Request, Response from .status_codes import codes from .hooks import dispatch_hook from .utils import cookiejar_from_dict, header_expand diff --git a/requests/models.py b/requests/models.py index d89f2fd2..50da2f68 100644 --- a/requests/models.py +++ b/requests/models.py @@ -82,7 +82,7 @@ class Request(object): self.response = Response() #: :class:`AuthObject` to attach to :class:`Request `. - # self.auth = auth + self.auth = auth #: CookieJar to attach to :class:`Request `. self.cookies = cookies @@ -161,7 +161,7 @@ class Request(object): ): # We already redirected. Don't keep it alive. - r.raw.close() + # r.raw.close() # Woah, this is getting crazy. if len(history) >= settings.max_redirects: @@ -196,7 +196,7 @@ class Request(object): # Create the new Request. request = Request( url, self.headers, self.files, method, - self.data, self.params, self.auth, self.cookiejar, + self.data, self.params, self.auth, self.cookies, # Flag as part of a redirect loop. redirect=True From dc7b4e7d32e45494d43d37c3d7c61032211b83ca Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 20:14:32 -0400 Subject: [PATCH 092/255] Don't add params to redirect / new configuration --- requests/config.py | 40 +++++++++++++++++++--------------------- requests/models.py | 16 ++++++++++++++-- requests/sessions.py | 14 ++++++++------ 3 files changed, 41 insertions(+), 29 deletions(-) diff --git a/requests/config.py b/requests/config.py index 55c1b88c..5bf49b45 100644 --- a/requests/config.py +++ b/requests/config.py @@ -8,8 +8,6 @@ This module provides the Requests settings feature set. settings parameters: -TODO: Verify!!! -TODO: Make sure format is acceptabl/cool - :base_headers: - Sets default User-Agent to `python-requests.org` - :accept_gzip: - Whether or not to accept gzip-compressed data - :proxies: - http proxies? @@ -18,29 +16,29 @@ TODO: Make sure format is acceptabl/cool - :max_redirects: - maximum number of allowed redirects? - :decode_unicode: - whether or not to accept unicode? -Used globally - """ +def merge_configs(config, default_config=None): + """Merge two given configurations.""" -class Settings(object): + # Use the module-level defaults, if none is given. + if default_config is None: + default_config = config.copy() - def __init__(self, **kwargs): - super(Settings, self).__init__() + d = default_config.copy() + d.update(config) - def __getattribute__(self, key): - return object.__getattribute__(self, key) + return d + +# Module-level defaults. +config = dict() + +config['base_headers'] = {'User-Agent': 'python-requests.org'} +config['accept_gzip'] = True +config['proxies'] = {} +config['verbose'] = None +config['timeout'] = None +config['max_redirects'] = 30 +config['decode_unicode'] = True -settings = Settings() - -settings.base_headers = {'User-Agent': 'python-requests.org'} -settings.accept_gzip = True -settings.proxies = None -settings.verbose = None -settings.timeout = None -settings.max_redirects = 30 -settings.decode_unicode = True - -#: Use socket.setdefaulttimeout() as fallback? -settings.timeout_fallback = True diff --git a/requests/models.py b/requests/models.py index 50da2f68..9ebd18d1 100644 --- a/requests/models.py +++ b/requests/models.py @@ -160,6 +160,10 @@ class Request(object): (self.allow_redirects)) ): + # print r.headers['location'] + # print dir(r.raw._original_response.fp) + # print '--' + # We already redirected. Don't keep it alive. # r.raw.close() @@ -195,8 +199,15 @@ class Request(object): # Create the new Request. request = Request( - url, self.headers, self.files, method, - self.data, self.params, self.auth, self.cookies, + url=url, + headers=self.headers, + files=self.files, + method=method, + data=self.data, + # params=self.params, + params=None, + auth=self.auth, + cookies=self.cookies, # Flag as part of a redirect loop. redirect=True @@ -272,6 +283,7 @@ class Request(object): # except (urllib2.HTTPError, urllib2.URLError), why: except Exception, why: + print why.__dict__ # if hasattr(why, 'reason'): # if isinstance(why.reason, socket.timeout): # why = Timeout(why) diff --git a/requests/sessions.py b/requests/sessions.py index 7145d278..910b1af8 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -18,12 +18,16 @@ from .utils import add_dict_to_cookiejar class Session(object): """A Requests session.""" - __attrs__ = ['headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks'] + __attrs__ = [ + 'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks', + 'config' + ] def __init__(self, **kwargs): # Set up a CookieJar to be used by default self.cookies = cookielib.FileCookieJar() + self.config = kwargs.get('config') or dict() # Map args from kwargs to instance-local variables map(lambda k, v: (k in self.__attrs__) and setattr(self, k, v), @@ -39,7 +43,6 @@ class Session(object): return self def __exit__(self, *args): - # print args pass def _map_api_methods(self): @@ -50,8 +53,8 @@ class Session(object): def pass_args(func): def wrapper_func(*args, **kwargs): - inst_attrs = dict((k, v) for k, v in self.__dict__.iteritems() - if k in self.__attrs__) + inst_attrs = dict((k, v) for k, v in self.__dict__.iteritems() if k in self.__attrs__) + # Combine instance-local values with kwargs values, with # priority to values in kwargs kwargs = dict(inst_attrs.items() + kwargs.items()) @@ -70,8 +73,7 @@ class Session(object): return wrapper_func # Map and decorate each function available in requests.api - map(lambda fn: setattr(self, fn, pass_args(getattr(api, fn))), - api.__all__) + map(lambda fn: setattr(self, fn, pass_args(getattr(api, fn))), api.__all__) def session(**kwargs): From 0cf0bde15853a704ed3fb4fd928187bdc90a3e9a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 20:36:30 -0400 Subject: [PATCH 093/255] session work merge kwargs and such --- requests/sessions.py | 59 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index 910b1af8..a5c13fbd 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -12,9 +12,30 @@ requests (cookies, auth, proxies). import cookielib from . import api +from .config import get_config from .utils import add_dict_to_cookiejar +def merge_kwargs(local_kwargs, default_kwargs): + """Merges kwarg dictionaries. + """ + + # Bypass if not a dictionary (e.g. timeout) + if not hasattr(local_kwargs, 'items'): + return local_kwargs + + kwargs = default_kwargs.copy() + kwargs.update(local_kwargs) + + # Remove keys that are set to None. + for (k,v) in local_kwargs.items(): + if v is None: + del kwargs[k] + + return kwargs + + + class Session(object): """A Requests session.""" @@ -23,19 +44,39 @@ class Session(object): 'config' ] - def __init__(self, **kwargs): + def __init__(self, + headers=None, + cookies=None, + auth=None, + timeout=None, + proxies=None, + hooks=None, + config=None): # Set up a CookieJar to be used by default - self.cookies = cookielib.FileCookieJar() - self.config = kwargs.get('config') or dict() + # self.cookies = cookielib.FileCookieJar() + # self.config = kwargs.get('config') + # self.configs = + self.headers = headers + self.cookies = cookies + self.auth = auth + self.timeout = timeout + self.proxies = proxies + self.hooks = hooks + self.config = get_config(config) + # print self.config # Map args from kwargs to instance-local variables - map(lambda k, v: (k in self.__attrs__) and setattr(self, k, v), - kwargs.iterkeys(), kwargs.itervalues()) + # map(lambda k, v: (k in self.__attrs__) and setattr(self, k, v), + # kwargs.iterkeys(), kwargs.itervalues()) # Map and wrap requests.api methods self._map_api_methods() + + def get(url, **kwargs): + + def __repr__(self): return '' % (id(self)) @@ -61,10 +102,10 @@ class Session(object): # If a session request has a cookie_dict, inject the # values into the existing CookieJar instead. - if isinstance(kwargs.get('cookies', None), dict): - kwargs['cookies'] = add_dict_to_cookiejar( - inst_attrs['cookies'], kwargs['cookies'] - ) + # if isinstance(kwargs.get('cookies', None), dict): + # kwargs['cookies'] = add_dict_to_cookiejar( + # inst_attrs['cookies'], kwargs['cookies'] + # ) if kwargs.get('headers', None) and inst_attrs.get('headers', None): kwargs['headers'].update(inst_attrs['headers']) From 351a711ce9a9771f61555b5057d4f249d9afc423 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 20:36:39 -0400 Subject: [PATCH 094/255] cleanup --- requests/config.py | 31 +++++++++++++++++++------------ requests/core.py | 1 - requests/models.py | 9 ++++----- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/requests/config.py b/requests/config.py index 5bf49b45..a9cfbec8 100644 --- a/requests/config.py +++ b/requests/config.py @@ -4,7 +4,7 @@ requests.config ~~~~~~~~~~~~~~~ -This module provides the Requests settings feature set. +This module provides the Requests configuration defaults. settings parameters: @@ -18,27 +18,34 @@ settings parameters: """ -def merge_configs(config, default_config=None): - """Merge two given configurations.""" +def get_config(config=None, default_config=None): + """Merges two given configurations.""" + + # Allow raw calls. + if config is None: + config=dict() # Use the module-level defaults, if none is given. if default_config is None: - default_config = config.copy() + default_config = defaults.copy() + d = default_config.copy() d.update(config) return d + # Module-level defaults. -config = dict() +defaults = dict() -config['base_headers'] = {'User-Agent': 'python-requests.org'} -config['accept_gzip'] = True -config['proxies'] = {} -config['verbose'] = None -config['timeout'] = None -config['max_redirects'] = 30 -config['decode_unicode'] = True +defaults['base_headers'] = {'User-Agent': 'python-requests.org'} +defaults['accept_gzip'] = True +defaults['proxies'] = {} +defaults['verbose'] = None +defaults['timeout'] = None +defaults['max_redirects'] = 30 +defaults['decode_unicode'] = True +defaults['keepalive'] = True diff --git a/requests/core.py b/requests/core.py index 477f39d7..0278f337 100644 --- a/requests/core.py +++ b/requests/core.py @@ -20,7 +20,6 @@ __copyright__ = 'Copyright 2011 Kenneth Reitz' from api import * -from config import settings from exceptions import * from models import Request, Response from sessions import session diff --git a/requests/models.py b/requests/models.py index 9ebd18d1..4f4ee942 100644 --- a/requests/models.py +++ b/requests/models.py @@ -7,18 +7,15 @@ requests.models """ import urllib -import urllib2 -import socket import zlib from urlparse import urlparse, urlunparse, urljoin -from datetime import datetime from .packages import urllib3 # print dir(urllib3) -from .config import settings +from .config import get_config from .structures import CaseInsensitiveDict from .utils import * from .status_codes import codes @@ -36,7 +33,7 @@ class Request(object): def __init__(self, url=None, headers=dict(), files=None, method=None, data=dict(), params=dict(), auth=None, cookies=None, timeout=None, redirect=False, - allow_redirects=False, proxies=None): + allow_redirects=False, proxies=None, config=None): #: Float describ the timeout of the request. # (Use socket.setdefaulttimeout() as fallback) @@ -72,6 +69,8 @@ class Request(object): # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) self.proxies = proxies + self.config = get_config(config) + self.data = data self._enc_data = encode_params(data) self.params = params From 802bcc9ba3f21f65d5eab48588b930222845ce23 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 20:57:29 -0400 Subject: [PATCH 095/255] configuration rework --- requests/{config.py => _config.py} | 0 requests/api.py | 9 ++++---- requests/models.py | 8 +++---- requests/sessions.py | 35 +++++++++++++++++++++++------- 4 files changed, 36 insertions(+), 16 deletions(-) rename requests/{config.py => _config.py} (100%) diff --git a/requests/config.py b/requests/_config.py similarity index 100% rename from requests/config.py rename to requests/_config.py diff --git a/requests/api.py b/requests/api.py index 1e4bd688..e08e530a 100644 --- a/requests/api.py +++ b/requests/api.py @@ -11,7 +11,7 @@ This module impliments the Requests API. """ -import config +from ._config import get_config from .models import Request, Response from .status_codes import codes from .hooks import dispatch_hook @@ -24,7 +24,7 @@ __all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete') def request(method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=False, proxies=None, hooks=None, - _connection=None): + config=None, _connection=None): """Constructs and sends a :class:`Request `. Returns :class:`Response ` object. @@ -44,6 +44,7 @@ def request(method, url, """ method = str(method).upper() + config = get_config(config) if cookies is None: cookies = {} @@ -64,9 +65,9 @@ def request(method, url, cookies=cookies, files=files, auth=auth, - timeout=timeout or config.settings.timeout, + timeout=timeout or config.get('timeout'), allow_redirects=allow_redirects, - proxies=proxies or config.settings.proxies, + proxies=proxies or config.get('proxies'), ) # Arguments manipulation hook. diff --git a/requests/models.py b/requests/models.py index 4f4ee942..a5bfc8d5 100644 --- a/requests/models.py +++ b/requests/models.py @@ -15,7 +15,7 @@ from urlparse import urlparse, urlunparse, urljoin from .packages import urllib3 # print dir(urllib3) -from .config import get_config +from ._config import get_config from .structures import CaseInsensitiveDict from .utils import * from .status_codes import codes @@ -91,15 +91,15 @@ class Request(object): # Header manipulation and defaults. - if settings.accept_gzip: - settings.base_headers.update({'Accept-Encoding': 'gzip'}) + if self.config.get('accept_gzip'): + self.headers.update({'Accept-Encoding': 'gzip'}) if headers: headers = CaseInsensitiveDict(self.headers) else: headers = CaseInsensitiveDict() - for (k, v) in settings.base_headers.items(): + for (k, v) in self.config.get('base_headers').items(): if k not in headers: headers[k] = v diff --git a/requests/sessions.py b/requests/sessions.py index a5c13fbd..56ccb850 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -12,23 +12,31 @@ requests (cookies, auth, proxies). import cookielib from . import api -from .config import get_config +from ._config import get_config from .utils import add_dict_to_cookiejar -def merge_kwargs(local_kwargs, default_kwargs): +def merge_kwargs(local_kwarg, default_kwarg): """Merges kwarg dictionaries. + + If a key in the dictionary is set to None, i """ # Bypass if not a dictionary (e.g. timeout) - if not hasattr(local_kwargs, 'items'): - return local_kwargs + if not hasattr(local_kwarg, 'items'): + return local_kwarg - kwargs = default_kwargs.copy() - kwargs.update(local_kwargs) + kwargs = default_kwarg.copy() + kwargs.update(local_kwarg) + + # from clint.textui import colored + + + # print colored.red(default_kwarg) + # print colored.red(local_kwarg) # Remove keys that are set to None. - for (k,v) in local_kwargs.items(): + for (k,v) in local_kwarg.items(): if v is None: del kwargs[k] @@ -74,8 +82,19 @@ class Session(object): self._map_api_methods() - def get(url, **kwargs): + def get(self, url, **kwargs): + _kwargs = {} + for attr in self.__attrs__: + default_attr = getattr(self, attr) + local_attr = kwargs.get(attr) + + new_attr = merge_kwargs(local_attr, default_attr) + + if new_attr is not None: + _kwargs[attr] = new_attr + + return api.get(url, **_kwargs) def __repr__(self): return '' % (id(self)) From ed160f2d2a25c7f8d307680532751cab3aa0c702 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 21:02:50 -0400 Subject: [PATCH 096/255] rework session argument Closes #141 --- requests/sessions.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index 56ccb850..7e5fa61f 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -113,23 +113,30 @@ class Session(object): def pass_args(func): def wrapper_func(*args, **kwargs): - inst_attrs = dict((k, v) for k, v in self.__dict__.iteritems() if k in self.__attrs__) - # Combine instance-local values with kwargs values, with - # priority to values in kwargs - kwargs = dict(inst_attrs.items() + kwargs.items()) + # Argument collector. + _kwargs = {} - # If a session request has a cookie_dict, inject the - # values into the existing CookieJar instead. - # if isinstance(kwargs.get('cookies', None), dict): - # kwargs['cookies'] = add_dict_to_cookiejar( - # inst_attrs['cookies'], kwargs['cookies'] - # ) + # Merge local and session arguments. + for attr in self.__attrs__: + default_attr = getattr(self, attr) + local_attr = kwargs.get(attr) - if kwargs.get('headers', None) and inst_attrs.get('headers', None): - kwargs['headers'].update(inst_attrs['headers']) + # Merge local and session dictionaries. + new_attr = merge_kwargs(local_attr, default_attr) - return func(*args, **kwargs) + # Skip attributes that were set to None. + if new_attr is not None: + _kwargs[attr] = new_attr + + # Make sure we didn't miss anything. + for (k, v) in kwargs: + if k not in _kwargs: + _kwargs[k] = v + + # TODO: Persist cookies. + + return func(*args, **_kwargs) return wrapper_func # Map and decorate each function available in requests.api From 7881aaa759ce05fa944a1530db8ba889ffaf68e4 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 21:04:13 -0400 Subject: [PATCH 097/255] =?UTF-8?q?clarity=E2=84=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requests/sessions.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index 7e5fa61f..b6c4db70 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -19,22 +19,17 @@ from .utils import add_dict_to_cookiejar def merge_kwargs(local_kwarg, default_kwarg): """Merges kwarg dictionaries. - If a key in the dictionary is set to None, i + If a local key in the dictionary is set to None, it will be removed. """ # Bypass if not a dictionary (e.g. timeout) if not hasattr(local_kwarg, 'items'): return local_kwarg + # Update new values. kwargs = default_kwarg.copy() kwargs.update(local_kwarg) - # from clint.textui import colored - - - # print colored.red(default_kwarg) - # print colored.red(local_kwarg) - # Remove keys that are set to None. for (k,v) in local_kwarg.items(): if v is None: @@ -130,7 +125,7 @@ class Session(object): _kwargs[attr] = new_attr # Make sure we didn't miss anything. - for (k, v) in kwargs: + for (k, v) in kwargs.items(): if k not in _kwargs: _kwargs[k] = v From 1758070cdc0ab3757f67f7eff0efecc0a1e753c9 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 21:06:41 -0400 Subject: [PATCH 098/255] clarity yo --- requests/sessions.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index b6c4db70..bb31e08f 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -56,10 +56,6 @@ class Session(object): hooks=None, config=None): - # Set up a CookieJar to be used by default - # self.cookies = cookielib.FileCookieJar() - # self.config = kwargs.get('config') - # self.configs = self.headers = headers self.cookies = cookies self.auth = auth @@ -67,13 +63,8 @@ class Session(object): self.proxies = proxies self.hooks = hooks self.config = get_config(config) - # print self.config - # Map args from kwargs to instance-local variables - # map(lambda k, v: (k in self.__attrs__) and setattr(self, k, v), - # kwargs.iterkeys(), kwargs.itervalues()) - - # Map and wrap requests.api methods + # Map and wrap requests.api methods. self._map_api_methods() From e2af50b7c9183b65766c6d84ad760c2a689f00ad Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 21:09:40 -0400 Subject: [PATCH 099/255] remove bunk get --- requests/sessions.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index bb31e08f..69306596 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -67,21 +67,6 @@ class Session(object): # Map and wrap requests.api methods. self._map_api_methods() - - def get(self, url, **kwargs): - - _kwargs = {} - for attr in self.__attrs__: - default_attr = getattr(self, attr) - local_attr = kwargs.get(attr) - - new_attr = merge_kwargs(local_attr, default_attr) - - if new_attr is not None: - _kwargs[attr] = new_attr - - return api.get(url, **_kwargs) - def __repr__(self): return '' % (id(self)) From ae4e85f11c39d0b2786a6b9df81c16cbfbe2a6e4 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 21:23:04 -0400 Subject: [PATCH 100/255] POOL PARTY!!!!!!!!!!!!!!! --- requests/_config.py | 1 + requests/api.py | 6 +++--- requests/models.py | 5 +++-- requests/sessions.py | 9 +++++++++ 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/requests/_config.py b/requests/_config.py index a9cfbec8..12aad6af 100644 --- a/requests/_config.py +++ b/requests/_config.py @@ -47,5 +47,6 @@ defaults['timeout'] = None defaults['max_redirects'] = 30 defaults['decode_unicode'] = True defaults['keepalive'] = True +defaults['max_connections'] = 10 diff --git a/requests/api.py b/requests/api.py index e08e530a..5debae1f 100644 --- a/requests/api.py +++ b/requests/api.py @@ -24,7 +24,7 @@ __all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete') def request(method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=False, proxies=None, hooks=None, - config=None, _connection=None): + config=None, _pools=None): """Constructs and sends a :class:`Request `. Returns :class:`Response ` object. @@ -40,7 +40,7 @@ def request(method, url, :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. - :param _connection: (optional) An HTTP Connection to re-use. + :param _pools: (optional) An HTTP PoolManager to use. """ method = str(method).upper() @@ -80,7 +80,7 @@ def request(method, url, r = dispatch_hook('pre_request', hooks, r) # Send the HTTP Request. - r.send(connection=_connection) + r.send(pools=_pools) # Post-request hook. r = dispatch_hook('post_request', hooks, r) diff --git a/requests/models.py b/requests/models.py index a5bfc8d5..820fc438 100644 --- a/requests/models.py +++ b/requests/models.py @@ -227,7 +227,7 @@ class Request(object): - def send(self, connection=None, anyway=False): + def send(self, pools=None, anyway=False): """Sends the shit.""" # Safety check. @@ -254,12 +254,13 @@ class Request(object): try: # Create a new HTTP connection, since one wasn't passed in. - if not connection: + if not pools: connection = urllib3.connection_from_url(url, timeout=self.timeout) # One-off request. Delay fetching the content until needed. do_block = False else: + connection = pools.connection_from_url(url, timeout=self.timeout) # Part of a connection pool, so no fancy stuff. Sorry! do_block = True diff --git a/requests/sessions.py b/requests/sessions.py index 69306596..38e6c501 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -14,6 +14,7 @@ import cookielib from . import api from ._config import get_config from .utils import add_dict_to_cookiejar +from .packages.urllib3.poolmanager import PoolManager def merge_kwargs(local_kwarg, default_kwarg): @@ -64,6 +65,10 @@ class Session(object): self.hooks = hooks self.config = get_config(config) + self.__pool = PoolManager( + num_pools=self.config.get('max_connections') + ) + # Map and wrap requests.api methods. self._map_api_methods() @@ -105,6 +110,10 @@ class Session(object): if k not in _kwargs: _kwargs[k] = v + # Add in PoolManager, if neccesary. + if self.config.get('keepalive'): + _kwargs['_pool'] = self.__pool + # TODO: Persist cookies. return func(*args, **_kwargs) From a699ddd14a305d933105de7900a079034bbc545f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 21:41:03 -0400 Subject: [PATCH 101/255] :sparkles: connection pooling :sparkles: works :D --- requests/models.py | 9 ++++++--- requests/sessions.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/requests/models.py b/requests/models.py index 820fc438..7604e2b6 100644 --- a/requests/models.py +++ b/requests/models.py @@ -122,6 +122,7 @@ class Request(object): def build(resp): response = Response() + response.config = self.config response.status_code = getattr(resp, 'status', None) try: @@ -236,7 +237,6 @@ class Request(object): # Build the final URL. url = build_url(self.url, self.params) - print url # Setup Files. if self.files: pass @@ -260,7 +260,7 @@ class Request(object): # One-off request. Delay fetching the content until needed. do_block = False else: - connection = pools.connection_from_url(url, timeout=self.timeout) + connection = pools.connection_from_url(url) # Part of a connection pool, so no fancy stuff. Sorry! do_block = True @@ -344,6 +344,9 @@ class Response(object): #: A dictionary of Cookies the server sent back. self.cookies = None + #: A dictionary of configuration. + self.config = None + def __repr__(self): return '' % (self.status_code) @@ -407,7 +410,7 @@ class Response(object): pass # Decode unicode content. - if settings.decode_unicode: + if self.config.get('decode_unicode'): self._content = get_unicode_from_response(self) self._content_consumed = True diff --git a/requests/sessions.py b/requests/sessions.py index 38e6c501..993d8bbe 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -112,7 +112,7 @@ class Session(object): # Add in PoolManager, if neccesary. if self.config.get('keepalive'): - _kwargs['_pool'] = self.__pool + _kwargs['_pools'] = self.__pool # TODO: Persist cookies. From dfe0e9382d69f0ae995c7499b73efe2426881e10 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 21:44:24 -0400 Subject: [PATCH 102/255] explaining --- requests/models.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/requests/models.py b/requests/models.py index 7604e2b6..e0391c54 100644 --- a/requests/models.py +++ b/requests/models.py @@ -122,15 +122,18 @@ class Request(object): def build(resp): response = Response() + + # Pass settings over. response.config = self.config + + # Fallback to None if there's no staus_code, for whatever reason. response.status_code = getattr(resp, 'status', None) - try: - response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) - response.raw = resp + # Make headers case-insensitive. + response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) - except AttributeError: - pass + # Save original resopnse for later. + response.raw = resp if is_error: response.error = resp From f52c0032a325b593f6e9fc0485ad6bd0ce871f12 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 21:46:07 -0400 Subject: [PATCH 103/255] urllib3 update --- requests/packages/urllib3/connectionpool.py | 18 ++++++++++++---- requests/packages/urllib3/poolmanager.py | 24 ++++++++++++++------- requests/packages/urllib3/response.py | 1 + 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 20770260..95e3dd4a 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -29,6 +29,8 @@ from .exceptions import ( log = logging.getLogger(__name__) +_Default = object() + ## Connection objects (extension of httplib) @@ -178,15 +180,19 @@ class HTTPConnectionPool(ConnectionPool): log.warning("HttpConnectionPool is full, discarding connection: %s" % self.host) - def _make_request(self, conn, method, url, **httplib_request_kw): + def _make_request(self, conn, method, url, timeout=_Default, + **httplib_request_kw): """ Perform a request on a given httplib connection object taken from our pool. """ self.num_requests += 1 + if timeout is _Default: + timeout = self.timeout + conn.request(method, url, **httplib_request_kw) - conn.sock.settimeout(self.timeout) + conn.sock.settimeout(timeout) httplib_response = conn.getresponse() log.debug("\"%s %s %s\" %s %s" % @@ -202,8 +208,8 @@ class HTTPConnectionPool(ConnectionPool): get_host(url) == (self.scheme, self.host, self.port)) def urlopen(self, method, url, body=None, headers=None, retries=3, - redirect=True, assert_same_host=True, pool_timeout=None, - release_conn=None, **response_kw): + redirect=True, assert_same_host=True, timeout=_Default, + pool_timeout=None, release_conn=None, **response_kw): """ Get a connection from the pool and perform an HTTP request. @@ -233,6 +239,9 @@ class HTTPConnectionPool(ConnectionPool): consistent else will raise HostChangedError. When False, you can use the pool on an HTTP proxy and request foreign hosts. + timeout + If specified, overrides the default timeout for this one request. + pool_timeout If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no @@ -273,6 +282,7 @@ class HTTPConnectionPool(ConnectionPool): try: # Make the request on the httplib connection object httplib_response = self._make_request(conn, method, url, + timeout=timeout, body=body, headers=headers) # Import httplib's response into our own wrapper object response = HTTPResponse.from_httplib(httplib_response, diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index d451f68a..18636967 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -34,18 +34,14 @@ class PoolManager(object): self.pools = RecentlyUsedContainer(num_pools) self.recently_used_pools = [] - def connection_from_url(self, url): + def connection_from_host(self, host, port=80, scheme='http'): """ - Similar to connectionpool.connection_from_url but doesn't pass any - additional keywords to the ConnectionPool constructor. Additional - keywords are taken from the PoolManager constructor. + Get a ConnectionPool based on the host, port, and scheme. """ - scheme, host, port = get_host(url) + pool_key = (scheme, host, port) # If the scheme, host, or port doesn't match existing open connections, # open a new ConnectionPool. - pool_key = (scheme, host, port or port_by_scheme.get(scheme, 80)) - pool = self.pools.get(pool_key) if pool: return pool @@ -58,7 +54,19 @@ class PoolManager(object): return pool + def connection_from_url(self, url): + """ + Similar to connectionpool.connection_from_url but doesn't pass any + additional keywords to the ConnectionPool constructor. Additional + keywords are taken from the PoolManager constructor. + """ + scheme, host, port = get_host(url) + + port = port or port_by_scheme.get(scheme, 80) + + return self.connection_from_host(host, port=port, scheme=scheme) + def urlopen(self, method, url, **kw): - "Same as HTTP(S)ConnectionPool.urlopen" + "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." conn = self.connection_from_url(url) return conn.urlopen(method, url, **kw) diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 14b30682..8c847ce1 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -83,6 +83,7 @@ class HTTPResponse(object): return self._pool._put_conn(self._connection) + self._connection = None @property def data(self): From cfc19b4ddea93926d99619d9cb6e3db68f6c5fc3 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 21:57:21 -0400 Subject: [PATCH 104/255] urllib3, don't decode content --- requests/models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index e0391c54..adca38fa 100644 --- a/requests/models.py +++ b/requests/models.py @@ -135,6 +135,7 @@ class Request(object): # Save original resopnse for later. response.raw = resp + # TODO: ? if is_error: response.error = resp @@ -277,7 +278,8 @@ class Request(object): assert_same_host=False, # preload_content=True # preload_content=False - preload_content=do_block + preload_content=do_block, + decode_content=False ) # Extract cookies. @@ -431,7 +433,6 @@ class Response(object): if (self.status_code >= 300) and (self.status_code < 400): raise Exception('300 yo') - elif (self.status_code >= 400) and (self.status_code < 500): raise Exception('400 yo') From cff3b7f9bd96b6837914ebb0fcf79ba874569ca5 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 25 Sep 2011 22:59:20 -0400 Subject: [PATCH 105/255] urllib3 update --- requests/packages/urllib3/connectionpool.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 95e3dd4a..add6fbc7 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -284,6 +284,7 @@ class HTTPConnectionPool(ConnectionPool): httplib_response = self._make_request(conn, method, url, timeout=timeout, body=body, headers=headers) + # Import httplib's response into our own wrapper object response = HTTPResponse.from_httplib(httplib_response, pool=self, @@ -309,7 +310,8 @@ class HTTPConnectionPool(ConnectionPool): finally: if release_conn: # Put the connection back to be reused - self._put_conn(conn) + response.release_conn() # Equivalent to self._put_conn(conn) but + # tracks release state. if not conn: log.warn("Retrying (%d attempts remain) after connection " From 111336459e408778d2915ba61a24c9c961e9aa54 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 26 Sep 2011 00:01:53 -0400 Subject: [PATCH 106/255] perfection. --- requests/api.py | 3 ++- requests/core.py | 2 ++ requests/models.py | 43 +++++++++++++++++++++++++++++-------------- requests/sessions.py | 7 ++++--- 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/requests/api.py b/requests/api.py index 5debae1f..653f82d3 100644 --- a/requests/api.py +++ b/requests/api.py @@ -68,6 +68,7 @@ def request(method, url, timeout=timeout or config.get('timeout'), allow_redirects=allow_redirects, proxies=proxies or config.get('proxies'), + _pools=_pools ) # Arguments manipulation hook. @@ -80,7 +81,7 @@ def request(method, url, r = dispatch_hook('pre_request', hooks, r) # Send the HTTP Request. - r.send(pools=_pools) + r.send() # Post-request hook. r = dispatch_hook('post_request', hooks, r) diff --git a/requests/core.py b/requests/core.py index 0278f337..e111c699 100644 --- a/requests/core.py +++ b/requests/core.py @@ -18,6 +18,8 @@ __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' +import logging +logging.basicConfig() from api import * from exceptions import * diff --git a/requests/models.py b/requests/models.py index adca38fa..adf29275 100644 --- a/requests/models.py +++ b/requests/models.py @@ -20,6 +20,7 @@ from .structures import CaseInsensitiveDict from .utils import * from .status_codes import codes from .exceptions import RequestException, Timeout, URLRequired, TooManyRedirects +from .packages.urllib3.poolmanager import PoolManager REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) @@ -33,7 +34,7 @@ class Request(object): def __init__(self, url=None, headers=dict(), files=None, method=None, data=dict(), params=dict(), auth=None, cookies=None, timeout=None, redirect=False, - allow_redirects=False, proxies=None, config=None): + allow_redirects=False, proxies=None, config=None, _pools=None): #: Float describ the timeout of the request. # (Use socket.setdefaulttimeout() as fallback) @@ -43,10 +44,10 @@ class Request(object): self.url = url #: Dictonary of HTTP Headers to attach to the :class:`Request `. - self.headers = headers + self.headers = headers or {} #: Dictionary of files to multipart upload (``{filename: content}``). - self.files = files + self.files = files or {} #: HTTP Method to use. self.method = method @@ -105,6 +106,8 @@ class Request(object): self.headers = headers + self._pools = _pools + def __repr__(self): return '' % (self.method) @@ -164,15 +167,11 @@ class Request(object): (self.allow_redirects)) ): - # print r.headers['location'] - # print dir(r.raw._original_response.fp) - # print '--' - # We already redirected. Don't keep it alive. # r.raw.close() # Woah, this is getting crazy. - if len(history) >= settings.max_redirects: + if len(history) >= self.config.get('max_redirects'): raise TooManyRedirects() # Add the old request to the history collector. @@ -212,6 +211,8 @@ class Request(object): params=None, auth=self.auth, cookies=self.cookies, + _pools=self._pools, + config=self.config, # Flag as part of a redirect loop. redirect=True @@ -232,7 +233,7 @@ class Request(object): - def send(self, pools=None, anyway=False): + def send(self, anyway=False): """Sends the shit.""" # Safety check. @@ -258,13 +259,25 @@ class Request(object): try: # Create a new HTTP connection, since one wasn't passed in. - if not pools: - connection = urllib3.connection_from_url(url, timeout=self.timeout) + if not self._pools: + # Create a pool manager for this one connection. + pools = PoolManager( + num_pools=self.config.get('max_connections'), + maxsize=1 + ) + + # Create a connection. + connection = pools.connection_from_url(url, timeout=self.timeout) # One-off request. Delay fetching the content until needed. do_block = False else: - connection = pools.connection_from_url(url) + # Create a connection. + connection = self._pools.connection_from_url(url) + + # Syntax sugar. + pools = self._pools + # Part of a connection pool, so no fancy stuff. Sorry! do_block = True @@ -276,12 +289,14 @@ class Request(object): headers=self.headers, redirect=False, assert_same_host=False, - # preload_content=True - # preload_content=False preload_content=do_block, decode_content=False ) + # Set the pools manager for redirections, if allowed. + if self.config.get('keepalive') and pools: + self._pools = pools + # Extract cookies. # if self.cookiejar is not None: # self.cookiejar.extract_cookies(resp, req) diff --git a/requests/sessions.py b/requests/sessions.py index 993d8bbe..83bfb1ea 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -65,8 +65,9 @@ class Session(object): self.hooks = hooks self.config = get_config(config) - self.__pool = PoolManager( - num_pools=self.config.get('max_connections') + self.__pools = PoolManager( + num_pools=10, + maxsize=1 ) # Map and wrap requests.api methods. @@ -112,7 +113,7 @@ class Session(object): # Add in PoolManager, if neccesary. if self.config.get('keepalive'): - _kwargs['_pools'] = self.__pool + _kwargs['_pools'] = self.__pools # TODO: Persist cookies. From a7d280cb3d9e24ceb9330b7157cf4e4d985c7a60 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 26 Sep 2011 00:04:38 -0400 Subject: [PATCH 107/255] cleanup --- requests/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index adf29275..6b882802 100644 --- a/requests/models.py +++ b/requests/models.py @@ -260,11 +260,11 @@ class Request(object): try: # Create a new HTTP connection, since one wasn't passed in. if not self._pools: + # Create a pool manager for this one connection. pools = PoolManager( num_pools=self.config.get('max_connections'), - maxsize=1 - ) + maxsize=1) # Create a connection. connection = pools.connection_from_url(url, timeout=self.timeout) From 1c5ce723d8ac41a97209c8c936d218c28f682157 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 26 Sep 2011 00:38:57 -0400 Subject: [PATCH 108/255] multipart file uplaods --- requests/models.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/requests/models.py b/requests/models.py index 6b882802..39d95e69 100644 --- a/requests/models.py +++ b/requests/models.py @@ -8,12 +8,10 @@ requests.models import urllib import zlib - from urlparse import urlparse, urlunparse, urljoin - from .packages import urllib3 -# print dir(urllib3) +from .packages.urllib3.filepost import encode_multipart_formdata from ._config import get_config from .structures import CaseInsensitiveDict @@ -242,13 +240,24 @@ class Request(object): # Build the final URL. url = build_url(self.url, self.params) - # Setup Files. + body = None + content_type = None + + # Multi-part file uploads. if self.files: - pass + if not isinstance(self.data, basestring): + fields = self.data.copy() + for (k, v) in self.files.items(): + fields.update({k: (None, v.read())) + (body, content_type) = encode_multipart_formdata(fields) # Setup form data. - elif self.data: - pass + if self.data and (not body): + if isinstance(self.data, basestring): + body = self.data + else: + body = urlencode(self.data) + content_type = 'application/x-www-form-urlencoded' # Setup cookies. elif self.cookies: @@ -264,10 +273,12 @@ class Request(object): # Create a pool manager for this one connection. pools = PoolManager( num_pools=self.config.get('max_connections'), - maxsize=1) + maxsize=1, + timeout=self.timeout + ) # Create a connection. - connection = pools.connection_from_url(url, timeout=self.timeout) + connection = pools.connection_from_url(url) # One-off request. Delay fetching the content until needed. do_block = False From 8ac2fe6eb170633b8b42e1af9b86e94729b339c2 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 26 Sep 2011 00:53:43 -0400 Subject: [PATCH 109/255] send file name --- requests/models.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index 39d95e69..01184f9d 100644 --- a/requests/models.py +++ b/requests/models.py @@ -248,7 +248,7 @@ class Request(object): if not isinstance(self.data, basestring): fields = self.data.copy() for (k, v) in self.files.items(): - fields.update({k: (None, v.read())) + fields.update({k: (k, v.read())}) (body, content_type) = encode_multipart_formdata(fields) # Setup form data. @@ -256,13 +256,16 @@ class Request(object): if isinstance(self.data, basestring): body = self.data else: - body = urlencode(self.data) + body = encode_params(self.data) content_type = 'application/x-www-form-urlencoded' # Setup cookies. elif self.cookies: pass + if (content_type) and (not 'content-type' in self.headers): + self.headers['Content-Type'] = content_type + # Only send the Request if new or forced. if (anyway) or (not self.sent): @@ -296,7 +299,7 @@ class Request(object): r = connection.urlopen( method=self.method, url=url, - body=self.data, + body=body, headers=self.headers, redirect=False, assert_same_host=False, From 2f9e287f7d1b9dfe5322c7c17a5a32a21a502aa0 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 26 Sep 2011 00:55:36 -0400 Subject: [PATCH 110/255] explicitly --- requests/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/requests/models.py b/requests/models.py index 01184f9d..406abd56 100644 --- a/requests/models.py +++ b/requests/models.py @@ -263,6 +263,7 @@ class Request(object): elif self.cookies: pass + # Add content-type if it wasn't explicitly provided. if (content_type) and (not 'content-type' in self.headers): self.headers['Content-Type'] = content_type From 14bd9ebb455ab1fd8a08868b37d4b271658b0dee Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 26 Sep 2011 01:33:18 -0400 Subject: [PATCH 111/255] yay cookies --- requests/api.py | 2 +- requests/models.py | 42 ++++++++++++++++++++++++++++++++++++++++-- requests/utils.py | 4 ++-- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/requests/api.py b/requests/api.py index 653f82d3..5e5bc299 100644 --- a/requests/api.py +++ b/requests/api.py @@ -49,7 +49,7 @@ def request(method, url, if cookies is None: cookies = {} - cookies = cookiejar_from_dict(cookies) + # cookies = cookiejar_from_dict(cookies) # Expand header values if headers: diff --git a/requests/models.py b/requests/models.py index 406abd56..101ecb4c 100644 --- a/requests/models.py +++ b/requests/models.py @@ -8,6 +8,7 @@ requests.models import urllib import zlib +from Cookie import SimpleCookie from urlparse import urlparse, urlunparse, urljoin from .packages import urllib3 @@ -133,6 +134,22 @@ class Request(object): # Make headers case-insensitive. response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) + # Start off with our local cookies. + cookies = self.cookies or dict() + + # Add new cookies from the server. + if 'set-cookie' in response.headers: + cookie_header = response.headers['set-cookie'] + + c = SimpleCookie() + c.load(cookie_header) + + for k,v in c.items(): + cookies.update({k: v.value}) + + # Save cookies in Response. + response.cookies = cookies + # Save original resopnse for later. response.raw = resp @@ -240,6 +257,7 @@ class Request(object): # Build the final URL. url = build_url(self.url, self.params) + # Nottin' on you. body = None content_type = None @@ -296,6 +314,20 @@ class Request(object): # Part of a connection pool, so no fancy stuff. Sorry! do_block = True + if self.cookies: + # Skip if 'cookie' header is explicitly set. + if 'cookie' not in self.headers: + + # Simple cookie with our dict. + c = SimpleCookie() + c.load(self.cookies) + + # Turn it into a header. + cookie_header = c.output(header='').strip() + + # Attach Cookie header to request. + self.headers['Cookie'] = cookie_header + # Create the connection. r = connection.urlopen( method=self.method, @@ -313,7 +345,14 @@ class Request(object): self._pools = pools # Extract cookies. - # if self.cookiejar is not None: + if self.cookies is not None: + pass + # cookies = cookiejar_from_dict(self.cookies) +# >>> C = Cookie.SimpleCookie() +# >>> C["rocky"] = "road" +# >>> C["rocky"]["path"] = "/cookie" +# >>> print C.output(header="Cookie:") +# Cookie: rocky=road; Path=/cookie # self.cookiejar.extract_cookies(resp, req) # except (urllib2.HTTPError, urllib2.URLError), why: @@ -332,7 +371,6 @@ class Request(object): self._build_response(r) self.response.ok = True - self.sent = self.response.ok return self.sent diff --git a/requests/utils.py b/requests/utils.py index 30ab9016..c422ac5c 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -117,7 +117,7 @@ def header_expand(headers): return ''.join(collector) -def dict_from_cookiejar(cj): +def dict_from_cookiejar(cookies): """Returns a key/value dictionary from a CookieJar. :param cj: CookieJar object to extract cookies from. @@ -125,7 +125,7 @@ def dict_from_cookiejar(cj): cookie_dict = {} - for _, cookies in cj._cookies.items(): + for _, cookies in cookies.items(): for _, cookies in cookies.items(): for cookie in cookies.values(): cookie_dict[cookie.name] = cookie.value From 5c132c98ca605ca390e2a2ddbb42f01f749fa484 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 26 Sep 2011 01:57:31 -0400 Subject: [PATCH 112/255] requests cookies behaving --- requests/models.py | 12 ++---------- requests/sessions.py | 21 ++++++++++++++++++--- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/requests/models.py b/requests/models.py index 101ecb4c..3f23ee3c 100644 --- a/requests/models.py +++ b/requests/models.py @@ -164,6 +164,7 @@ class Request(object): # Create the lone response object. r = build(resp) + self.cookies.update(r.cookies) # Store the HTTP response, just in case. r._response = resp @@ -236,6 +237,7 @@ class Request(object): # Send her away! request.send() r = request.response + self.cookies.update(r.cookies) # Insert collected history. r.history = history @@ -344,16 +346,6 @@ class Request(object): if self.config.get('keepalive') and pools: self._pools = pools - # Extract cookies. - if self.cookies is not None: - pass - # cookies = cookiejar_from_dict(self.cookies) -# >>> C = Cookie.SimpleCookie() -# >>> C["rocky"] = "road" -# >>> C["rocky"]["path"] = "/cookie" -# >>> print C.output(header="Cookie:") -# Cookie: rocky=road; Path=/cookie - # self.cookiejar.extract_cookies(resp, req) # except (urllib2.HTTPError, urllib2.URLError), why: except Exception, why: diff --git a/requests/sessions.py b/requests/sessions.py index 83bfb1ea..16f4fba4 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -58,7 +58,7 @@ class Session(object): config=None): self.headers = headers - self.cookies = cookies + self.cookies = cookies or {} self.auth = auth self.timeout = timeout self.proxies = proxies @@ -96,10 +96,18 @@ class Session(object): # Merge local and session arguments. for attr in self.__attrs__: + # if attr == 'cookies': + # print getattr(self, attr) + # print kwargs.get(attr) + + # Merge local and session dictionaries. default_attr = getattr(self, attr) local_attr = kwargs.get(attr) - # Merge local and session dictionaries. + # Cookies persist. + if attr == 'cookies': + local_attr = local_attr or {} + new_attr = merge_kwargs(local_attr, default_attr) # Skip attributes that were set to None. @@ -116,8 +124,15 @@ class Session(object): _kwargs['_pools'] = self.__pools # TODO: Persist cookies. + # print self.cookies - return func(*args, **_kwargs) + # print _kwargs.get('cookies') + # print '%%%' + + r = func(*args, **_kwargs) + # print r.cookies + self.cookies.update(r.cookies) + return r return wrapper_func # Map and decorate each function available in requests.api From 62d29d510625aa6c5415f4ba35579059b1c842fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Mon, 26 Sep 2011 10:57:45 +0200 Subject: [PATCH 113/255] Fixed #174 and refactored urls quoting/concatenation in one function in utils.py. --- requests/models.py | 30 +++--------------------------- requests/utils.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/requests/models.py b/requests/models.py index f6d84ebd..18ae26ab 100644 --- a/requests/models.py +++ b/requests/models.py @@ -22,7 +22,7 @@ from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicA from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers -from .utils import dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip +from .utils import get_clean_url, dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip from .status_codes import codes from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects @@ -215,20 +215,7 @@ class Request(object): history.append(r) - url = r.headers['location'] - - # Handle redirection without scheme (see: RFC 1808 Section 4) - if url.startswith('//'): - parsed_rurl = urlparse(r.url) - url = '%s:%s' % (parsed_rurl.scheme, url) - - # Facilitate non-RFC2616-compliant 'location' headers - # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') - parsed_url = urlparse(url) - if not parsed_url.netloc: - parsed_url = list(parsed_url) - parsed_url[2] = urllib.quote(parsed_url[2], safe="%/:=&?~#+!$,;'@()*[]") - url = urljoin(r.url, str(urlunparse(parsed_url))) + url = get_clean_url(r.headers['location'], parent_url=r.url) # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 if r.status_code is codes.see_other: @@ -276,18 +263,7 @@ class Request(object): def _build_url(self): """Build the actual URL to use.""" - # Support for unicode domain names and paths. - scheme, netloc, path, params, query, fragment = urlparse(self.url) - netloc = netloc.encode('idna') - - if isinstance(path, unicode): - path = path.encode('utf-8') - - path = urllib.quote(path, safe="%/:=&?~#+!$,;'@()*[]") - - self.url = str(urlunparse( - [scheme, netloc, path, params, query, fragment] - )) + self.url = get_clean_url(self.url) if self._enc_params: if urlparse(self.url).query: diff --git a/requests/utils.py b/requests/utils.py index 75357ae7..e5036e44 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -13,8 +13,36 @@ import cgi import codecs import cookielib import re +import urllib import zlib +from urlparse import urlparse, urlunparse, urljoin + +def get_clean_url(url, parent_url=None): + # Handle redirection without scheme (see: RFC 1808 Section 4) + if url.startswith('//'): + parsed_rurl = urlparse(parent_url) + url = '%s:%s' % (parsed_rurl.scheme, url) + + scheme, netloc, path, params, query, fragment = urlparse(url) + if netloc: + netloc = netloc.encode('idna') + + if isinstance(path, unicode): + path = path.encode('utf-8') + + path = urllib.quote(path, safe="%/:=&?~#+!$,;'@()*[]") + params = urllib.quote(params, safe="%/:=&?~#+!$,;'@()*[]") + query = urllib.quote(query, safe="%/:=&?~#+!$,;'@()*[]") + + url = str(urlunparse([scheme, netloc, path, params, query, fragment])) + + # Facilitate non-RFC2616-compliant 'location' headers + # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') + if not netloc and parent_url: + url = urljoin(parent_url, url) + + return url def header_expand(headers): """Returns an HTTP Header value string from a dictionary. From ffde764a910e64398d01a8058a540853b63d60d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Mon, 26 Sep 2011 11:00:25 +0200 Subject: [PATCH 114/255] Removed unused imports. --- requests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 18ae26ab..43b437ee 100644 --- a/requests/models.py +++ b/requests/models.py @@ -14,7 +14,7 @@ import zlib from urllib2 import HTTPError -from urlparse import urlparse, urlunparse, urljoin +from urlparse import urlparse from datetime import datetime from .config import settings From 87e8a12a9f0c9bfafabab2d48e5e90e05615f3c1 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 29 Sep 2011 10:28:00 -0300 Subject: [PATCH 115/255] Edited requests/api.py via GitHub --- requests/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/api.py b/requests/api.py index 48823a9a..56d825fd 100644 --- a/requests/api.py +++ b/requests/api.py @@ -4,7 +4,7 @@ requests.api ~~~~~~~~~~~~ -This module impliments the Requests API. +This module implements the Requests API. :copyright: (c) 2011 by Kenneth Reitz. :license: ISC, see LICENSE for more details. From 4cd54d2f8798928b31b67ff1a9361759a40aff5f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 30 Sep 2011 07:07:09 -0400 Subject: [PATCH 116/255] keepalive => keep_alive --- requests/_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/_config.py b/requests/_config.py index 12aad6af..7ecc2a10 100644 --- a/requests/_config.py +++ b/requests/_config.py @@ -46,7 +46,7 @@ defaults['verbose'] = None defaults['timeout'] = None defaults['max_redirects'] = 30 defaults['decode_unicode'] = True -defaults['keepalive'] = True +defaults['keep_alive'] = True defaults['max_connections'] = 10 From 2f5008107c56485a46271062ecc0b75f5e49fa16 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 30 Sep 2011 07:07:39 -0400 Subject: [PATCH 117/255] keepalive => keep_alive --- requests/sessions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index 16f4fba4..cfb73695 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -120,7 +120,7 @@ class Session(object): _kwargs[k] = v # Add in PoolManager, if neccesary. - if self.config.get('keepalive'): + if self.config.get('keep_alive'): _kwargs['_pools'] = self.__pools # TODO: Persist cookies. @@ -131,7 +131,7 @@ class Session(object): r = func(*args, **_kwargs) # print r.cookies - self.cookies.update(r.cookies) + self.cookies.update(r.cookies or {}) return r return wrapper_func From 2e391fa44cb1d5d2d5c4321c547d470d0b594333 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 30 Sep 2011 07:08:04 -0400 Subject: [PATCH 118/255] pass hooks around, disable auto send request. --- requests/api.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/requests/api.py b/requests/api.py index 5e5bc299..9c753ffd 100644 --- a/requests/api.py +++ b/requests/api.py @@ -24,7 +24,7 @@ __all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete') def request(method, url, params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=False, proxies=None, hooks=None, - config=None, _pools=None): + config=None, _pools=None, _return_request=False): """Constructs and sends a :class:`Request `. Returns :class:`Response ` object. @@ -66,6 +66,7 @@ def request(method, url, files=files, auth=auth, timeout=timeout or config.get('timeout'), + hooks=hooks, allow_redirects=allow_redirects, proxies=proxies or config.get('proxies'), _pools=_pools @@ -80,9 +81,14 @@ def request(method, url, # Pre-request hook. r = dispatch_hook('pre_request', hooks, r) + # Only construct the request (for async) + if _return_request: + return r + # Send the HTTP Request. r.send() + # TODO: Add these hooks inline. # Post-request hook. r = dispatch_hook('post_request', hooks, r) From e8c2fc15852128a9da5f7db53ddd8f068624a71b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 30 Sep 2011 07:08:33 -0400 Subject: [PATCH 119/255] ch ch ch changes --- requests/models.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/requests/models.py b/requests/models.py index 3f23ee3c..5235f36e 100644 --- a/requests/models.py +++ b/requests/models.py @@ -33,7 +33,12 @@ class Request(object): def __init__(self, url=None, headers=dict(), files=None, method=None, data=dict(), params=dict(), auth=None, cookies=None, timeout=None, redirect=False, - allow_redirects=False, proxies=None, config=None, _pools=None): + allow_redirects=False, proxies=None, config=None, hooks=None, + _pools=None): + + if cookies is None: + cookies = {} + #: Float describ the timeout of the request. # (Use socket.setdefaulttimeout() as fallback) @@ -105,6 +110,7 @@ class Request(object): self.headers = headers + self.hooks = hooks self._pools = _pools def __repr__(self): @@ -237,7 +243,8 @@ class Request(object): # Send her away! request.send() r = request.response - self.cookies.update(r.cookies) + + self.cookies.update(r.cookies or {}) # Insert collected history. r.history = history @@ -280,8 +287,6 @@ class Request(object): content_type = 'application/x-www-form-urlencoded' # Setup cookies. - elif self.cookies: - pass # Add content-type if it wasn't explicitly provided. if (content_type) and (not 'content-type' in self.headers): @@ -343,7 +348,7 @@ class Request(object): ) # Set the pools manager for redirections, if allowed. - if self.config.get('keepalive') and pools: + if self.config.get('keep_alive') and pools: self._pools = pools From 30d937f299d217f2107139b3c0a8083d90d184aa Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 30 Sep 2011 07:08:38 -0400 Subject: [PATCH 120/255] urllib3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index de7dcb0e..783e0397 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( packages= [ 'requests', 'requests.packages', - 'requests.packages.poster' + 'requests.packages.urllib3' ], install_requires=required, license='ISC', From 3092483433ebf0f5802677aa9b8a74ee4dc1e64b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 30 Sep 2011 07:08:59 -0400 Subject: [PATCH 121/255] requests.async. Oh, yes :) --- requests/async.py | 75 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 requests/async.py diff --git a/requests/async.py b/requests/async.py new file mode 100644 index 00000000..78d3d9a4 --- /dev/null +++ b/requests/async.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +""" +reqeusts.async +~~~~~~~~~~~~~~ + +This module contains an asyncronous replica of ``requests.api``, powered +by gevent. All API methods return a ``Request`` instance (as opposed to +``Response``). A list of requests can be sent with ``map()``. +""" + +try: + import gevent + from gevent import monkey as curious_george +except ImportError: + raise RuntimeError('Gevent is required for requests.async.') + +# Monkey-patch. +curious_george.patch_all(thread=False) + +from . import api +from .hooks import dispatch_hook +from .packages.urllib3.poolmanager import PoolManager + + +def _patched(f): + """Patches a given api function to not send.""" + + def wrapped(*args, **kwargs): + return f(*args, _return_request=True, **kwargs) + + return wrapped + + +def _send(r, pools=None): + """Dispatcher.""" + + if pools: + r._pools = pools + + r.send() + + # Post-request hook. + r = dispatch_hook('post_request', r.hooks, r) + + # Response manipulation hook. + r.response = dispatch_hook('response', r.hooks, r.response) + + return r.response + +# Patched requests.api functions. +get = _patched(api.get) +head = _patched(api.head) +post = _patched(api.post) +put = _patched(api.put) +patch = _patched(api.patch) +delete = _patched(api.delete) +request = _patched(api.request) + +from requests.sessions import session + +def map(requests, keep_alive=False): + """Sends the requests... Asynchronously.""" + + if keep_alive: + pools = PoolManager(num_pools=len(requests), maxsize=1) + else: + pools = None + + jobs = [gevent.spawn(_send, r, pools=pools) for r in requests] + gevent.joinall(jobs) + + return [r.response for r in requests] + + From b58433fb74d040e7828a3a047a5ca25dba5751ab Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 1 Oct 2011 04:34:52 -0400 Subject: [PATCH 122/255] ascii arts --- requests/core.py | 6 ++++++ requests/exceptions.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/requests/core.py b/requests/core.py index e111c699..613e3324 100644 --- a/requests/core.py +++ b/requests/core.py @@ -1,5 +1,10 @@ # -*- coding: utf-8 -*- +# __ +# /__) _ _ _ _ _/ _ +# / ( (- (/ (/ (- _) / _) +# / + """ requests.core ~~~~~~~~~~~~~ @@ -11,6 +16,7 @@ This module implements the main Requests system. """ + __title__ = 'requests' __version__ = '0.6.2 (dev)' __build__ = 0x000602 diff --git a/requests/exceptions.py b/requests/exceptions.py index ecedd7f6..74a584c9 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -13,7 +13,7 @@ class RequestException(Exception): class AuthenticationError(RequestException): """The authentication credentials provided were invalid.""" - + class AuthenticationError(RequestException): """The authentication credentials provided were invalid.""" From 47b98bc90f84e1ebd2ca220843df5cb49f6db99a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 1 Oct 2011 04:36:50 -0400 Subject: [PATCH 123/255] Typos /cc @umbrae --- requests/async.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/async.py b/requests/async.py index 78d3d9a4..8d84bc90 100644 --- a/requests/async.py +++ b/requests/async.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- """ -reqeusts.async +requests.async ~~~~~~~~~~~~~~ -This module contains an asyncronous replica of ``requests.api``, powered +This module contains an asynchronous replica of ``requests.api``, powered by gevent. All API methods return a ``Request`` instance (as opposed to ``Response``). A list of requests can be sent with ``map()``. """ From 5afe11ba1c00680edaee91b92ba87486997965f7 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 1 Oct 2011 04:40:53 -0400 Subject: [PATCH 124/255] async module cleanups --- requests/async.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/requests/async.py b/requests/async.py index 8d84bc90..cfd19644 100644 --- a/requests/async.py +++ b/requests/async.py @@ -23,8 +23,14 @@ from .hooks import dispatch_hook from .packages.urllib3.poolmanager import PoolManager +__all__ = ( + 'map', 'get', 'head', 'post', 'put', 'patch', 'delete', 'request' +) + + + def _patched(f): - """Patches a given api function to not send.""" + """Patches a given API function to not send.""" def wrapped(*args, **kwargs): return f(*args, _return_request=True, **kwargs) @@ -33,7 +39,7 @@ def _patched(f): def _send(r, pools=None): - """Dispatcher.""" + """Sends a given Request object.""" if pools: r._pools = pools @@ -48,6 +54,7 @@ def _send(r, pools=None): return r.response + # Patched requests.api functions. get = _patched(api.get) head = _patched(api.head) @@ -57,10 +64,12 @@ patch = _patched(api.patch) delete = _patched(api.delete) request = _patched(api.request) -from requests.sessions import session def map(requests, keep_alive=False): - """Sends the requests... Asynchronously.""" + """Concurrently converts a list of Requests to Responses. + + :param requests: a collection of Request objects. + """ if keep_alive: pools = PoolManager(num_pools=len(requests), maxsize=1) @@ -73,3 +82,5 @@ def map(requests, keep_alive=False): return [r.response for r in requests] + + From 7875a12617ae38f3432779b29e0fdbc93febd408 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 1 Oct 2011 04:52:13 -0400 Subject: [PATCH 125/255] async documentation --- requests/async.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requests/async.py b/requests/async.py index cfd19644..9517064d 100644 --- a/requests/async.py +++ b/requests/async.py @@ -24,11 +24,11 @@ from .packages.urllib3.poolmanager import PoolManager __all__ = ( - 'map', 'get', 'head', 'post', 'put', 'patch', 'delete', 'request' + 'map', + 'get', 'head', 'post', 'put', 'patch', 'delete', 'request' ) - def _patched(f): """Patches a given API function to not send.""" @@ -69,6 +69,7 @@ def map(requests, keep_alive=False): """Concurrently converts a list of Requests to Responses. :param requests: a collection of Request objects. + :param keep_alive: If True, HTTP Keep-Alive will be used. """ if keep_alive: From c9f5614417fc2b997e2e5223d8bdacf07b9e13be Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 1 Oct 2011 06:04:18 -0400 Subject: [PATCH 126/255] urllib3 update --- requests/packages/urllib3/__init__.py | 6 ++++ requests/packages/urllib3/_collections.py | 6 ++++ requests/packages/urllib3/connectionpool.py | 33 ++++++++++++++----- requests/packages/urllib3/contrib/ntlmpool.py | 6 ++++ requests/packages/urllib3/exceptions.py | 6 ++++ requests/packages/urllib3/filepost.py | 6 ++++ requests/packages/urllib3/poolmanager.py | 9 ++++- requests/packages/urllib3/response.py | 6 ++++ 8 files changed, 69 insertions(+), 9 deletions(-) diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index 19a62391..5c4b5d17 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -1,3 +1,9 @@ +# urllib3/__init__.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + """ urllib3 - Thread-safe connection pooling and re-using. """ diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py index 0e1c8e69..2b0de0e8 100644 --- a/requests/packages/urllib3/_collections.py +++ b/requests/packages/urllib3/_collections.py @@ -1,3 +1,9 @@ +# urllib3/_collections.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + from collections import MutableMapping, deque diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index add6fbc7..92ce1540 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -1,3 +1,9 @@ +# urllib3/connectionpool.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + import logging import socket @@ -191,10 +197,12 @@ class HTTPConnectionPool(ConnectionPool): if timeout is _Default: timeout = self.timeout + conn.request(method, url, **httplib_request_kw) conn.sock.settimeout(timeout) httplib_response = conn.getresponse() + log.debug("\"%s %s %s\" %s %s" % (method, url, conn._http_vsn_str, # pylint: disable-msg=W0212 @@ -276,14 +284,17 @@ class HTTPConnectionPool(ConnectionPool): raise HostChangedError("Connection pool with host '%s' tried to " "open a foreign host: %s" % (host, url)) - # Request a connection from the queue - conn = self._get_conn(timeout=pool_timeout) + conn = None try: + # Request a connection from the queue + # (Could raise SocketError: Bad file descriptor) + conn = self._get_conn(timeout=pool_timeout) # Make the request on the httplib connection object httplib_response = self._make_request(conn, method, url, timeout=timeout, body=body, headers=headers) + # print '!' # Import httplib's response into our own wrapper object response = HTTPResponse.from_httplib(httplib_response, @@ -291,8 +302,14 @@ class HTTPConnectionPool(ConnectionPool): connection=conn, **response_kw) - # The connection will be put back into the pool when - # response.release_conn() is called (implicitly by response.read()) + if release_conn: + # The connection will be released manually in the ``finally:`` + response.connection = None + + # else: + # The connection will be put back into the pool when + # ``response.release_conn()`` is called (implicitly by + # ``response.read()``) except (SocketTimeout, Empty), e: # Timed out either by socket or queue @@ -308,10 +325,9 @@ class HTTPConnectionPool(ConnectionPool): conn = None finally: - if release_conn: + if conn and release_conn: # Put the connection back to be reused - response.release_conn() # Equivalent to self._put_conn(conn) but - # tracks release state. + self._put_conn(conn) if not conn: log.warn("Retrying (%d attempts remain) after connection " @@ -399,7 +415,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): strict=False, timeout=None, maxsize=1, block=False, headers=None, key_file=None, cert_file=None, - cert_reqs='CERT_NONE', ca_certs=None): + cert_reqs=ssl.CERT_REQUIRED, ca_certs=None): super(HTTPSConnectionPool, self).__init__(host, port, strict, timeout, maxsize, @@ -413,6 +429,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): """ Return a fresh HTTPSConnection. """ + self.num_connections += 1 log.info("Starting new HTTPS connection (%d): %s" % (self.num_connections, self.host)) diff --git a/requests/packages/urllib3/contrib/ntlmpool.py b/requests/packages/urllib3/contrib/ntlmpool.py index a4642fac..c5f010e1 100644 --- a/requests/packages/urllib3/contrib/ntlmpool.py +++ b/requests/packages/urllib3/contrib/ntlmpool.py @@ -1,3 +1,9 @@ +# urllib3/contrib/ntlmpool.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + """ NTLM authenticating pool, contributed by erikcederstran diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py index 45b0e822..69f459bd 100644 --- a/requests/packages/urllib3/exceptions.py +++ b/requests/packages/urllib3/exceptions.py @@ -1,3 +1,9 @@ +# urllib3/exceptions.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + ## Exceptions class HTTPError(Exception): diff --git a/requests/packages/urllib3/filepost.py b/requests/packages/urllib3/filepost.py index 67d2d0d8..8e65b5c2 100644 --- a/requests/packages/urllib3/filepost.py +++ b/requests/packages/urllib3/filepost.py @@ -1,3 +1,9 @@ +# urllib3/filepost.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + import codecs import mimetools import mimetypes diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index 18636967..d5b7613c 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -1,3 +1,9 @@ +# urllib3/poolmanager.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + from ._collections import RecentlyUsedContainer from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, get_host @@ -40,6 +46,7 @@ class PoolManager(object): """ pool_key = (scheme, host, port) + # If the scheme, host, or port doesn't match existing open connections, # open a new ConnectionPool. pool = self.pools.get(pool_key) @@ -64,7 +71,7 @@ class PoolManager(object): port = port or port_by_scheme.get(scheme, 80) - return self.connection_from_host(host, port=port, scheme=scheme) + return self.connection_from_host(host, port=port, scheme=scheme) def urlopen(self, method, url, **kw): "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 8c847ce1..2bbc06ab 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -1,3 +1,9 @@ +# urllib3/response.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + import gzip import logging import zlib From e1c0646f95930cfbd1b404aa820eda3f9836080f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 1 Oct 2011 06:04:41 -0400 Subject: [PATCH 127/255] urllib3 bugfix --- requests/packages/urllib3/connectionpool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 92ce1540..36ecfb51 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -46,8 +46,8 @@ class VerifiedHTTPSConnection(HTTPSConnection): SSL certification. """ - def __init__(self): - HTTPSConnection.__init__() + def __init__(self, **kwargs): + HTTPSConnection.__init__(self, **kwargs) self.cert_reqs = None self.ca_certs = None From 1997dee3d9f0933bca99b2065463ad0a7afc7f10 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 1 Oct 2011 06:17:02 -0400 Subject: [PATCH 128/255] :sparkles: :cake: :sparkles: --- requests/packages/urllib3/poolmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index d5b7613c..8d915ad5 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -15,7 +15,7 @@ pool_classes_by_scheme = { port_by_scheme = { 'http': 80, - 'https': 433, + 'https': 443, } From 0b475dd65487f2c14efa6f917657a0f1d19bae02 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 1 Oct 2011 20:51:41 -0400 Subject: [PATCH 129/255] Test improvements #180 --- requests/packages/urllib3/connectionpool.py | 1 + requests/packages/urllib3/poolmanager.py | 4 +- tests/integration/test_requests.py | 530 -------------------- tests/unit/test_requests_api.py | 209 -------- 4 files changed, 4 insertions(+), 740 deletions(-) delete mode 100755 tests/integration/test_requests.py delete mode 100755 tests/unit/test_requests_api.py diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 36ecfb51..8f74b1f5 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -38,6 +38,7 @@ log = logging.getLogger(__name__) _Default = object() + ## Connection objects (extension of httplib) class VerifiedHTTPSConnection(HTTPSConnection): diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index 8d915ad5..acf1c5a1 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -71,7 +71,9 @@ class PoolManager(object): port = port or port_by_scheme.get(scheme, 80) - return self.connection_from_host(host, port=port, scheme=scheme) + r = self.connection_from_host(host, port=port, scheme=scheme) + print r.__dict__ + return r def urlopen(self, method, url, **kw): "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." diff --git a/tests/integration/test_requests.py b/tests/integration/test_requests.py deleted file mode 100755 index 406d23c1..00000000 --- a/tests/integration/test_requests.py +++ /dev/null @@ -1,530 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from __future__ import with_statement - -import unittest -import cookielib - -try: - import omnijson as json -except ImportError: - import json - -import requests - -from requests.sessions import Session -from requests.utils import curl_from_request - - -HTTPBIN_URL = 'http://httpbin.org/' -HTTPSBIN_URL = 'https://httpbin.ep.io/' - -# HTTPBIN_URL = 'http://staging.httpbin.org/' -# HTTPSBIN_URL = 'https://httpbin-staging.ep.io/' - - -def httpbin(*suffix): - """Returns url for HTTPBIN resource.""" - - return HTTPBIN_URL + '/'.join(suffix) - - -def httpsbin(*suffix): - """Returns url for HTTPSBIN resource.""" - - return HTTPSBIN_URL + '/'.join(suffix) - - -SERVICES = (httpbin, httpsbin) - - - -class RequestsTestSuite(unittest.TestCase): - """Requests test cases.""" - - - def setUp(self): - pass - - - def tearDown(self): - """Teardown.""" - pass - - - def test_invalid_url(self): - self.assertRaises(ValueError, requests.get, 'hiwpefhipowhefopw') - - - def test_HTTP_200_OK_GET(self): - r = requests.get(httpbin('/')) - self.assertEqual(r.status_code, 200) - - def test_HTTP_302_ALLOW_REDIRECT_GET(self): - r = requests.get(httpbin('redirect', '1')) - self.assertEqual(r.status_code, 200) - - def test_HTTP_302_GET(self): - r = requests.get(httpbin('redirect', '1'), allow_redirects=False) - self.assertEqual(r.status_code, 302) - - def test_HTTPS_200_OK_GET(self): - r = requests.get(httpsbin('/')) - self.assertEqual(r.status_code, 200) - - - def test_HTTP_200_OK_GET_WITH_PARAMS(self): - heads = {'User-agent': 'Mozilla/5.0'} - - r = requests.get(httpbin('user-agent'), headers=heads) - - assert heads['User-agent'] in r.content - self.assertEqual(r.status_code, 200) - - - def test_HTTP_200_OK_GET_WITH_MIXED_PARAMS(self): - heads = {'User-agent': 'Mozilla/5.0'} - - r = requests.get(httpbin('get') + '?test=true', params={'q': 'test'}, headers=heads) - self.assertEqual(r.status_code, 200) - - - def test_user_agent_transfers(self): - """Issue XX""" - - heads = { - 'User-agent': - 'Mozilla/5.0 (github.com/kennethreitz/requests)' - } - - r = requests.get(httpbin('user-agent'), headers=heads); - self.assertTrue(heads['User-agent'] in r.content) - - heads = { - 'user-agent': - 'Mozilla/5.0 (github.com/kennethreitz/requests)' - } - - r = requests.get(httpbin('user-agent'), headers=heads); - self.assertTrue(heads['user-agent'] in r.content) - - - def test_HTTP_200_OK_HEAD(self): - r = requests.head(httpbin('/')) - self.assertEqual(r.status_code, 200) - - - def test_HTTPS_200_OK_HEAD(self): - r = requests.head(httpsbin('/')) - self.assertEqual(r.status_code, 200) - - - def test_HTTP_200_OK_PUT(self): - r = requests.put(httpbin('put')) - self.assertEqual(r.status_code, 200) - - - def test_HTTPS_200_OK_PUT(self): - r = requests.put(httpsbin('put')) - self.assertEqual(r.status_code, 200) - - - def test_HTTP_200_OK_PATCH(self): - r = requests.patch(httpbin('patch')) - self.assertEqual(r.status_code, 200) - - - def test_HTTPS_200_OK_PATCH(self): - r = requests.patch(httpsbin('patch')) - self.assertEqual(r.status_code, 200) - - - def test_AUTH_HTTP_200_OK_GET(self): - - for service in SERVICES: - - auth = ('user', 'pass') - url = service('basic-auth', 'user', 'pass') - - r = requests.get(url, auth=auth) - # print r.__dict__ - self.assertEqual(r.status_code, 200) - - - r = requests.get(url) - self.assertEqual(r.status_code, 200) - - - def test_POSTBIN_GET_POST_FILES(self): - - for service in SERVICES: - - url = service('post') - post = requests.post(url).raise_for_status() - - post = requests.post(url, data={'some': 'data'}) - self.assertEqual(post.status_code, 200) - - post2 = requests.post(url, files={'some': open('test_requests.py')}) - self.assertEqual(post2.status_code, 200) - - post3 = requests.post(url, data='[{"some": "json"}]') - self.assertEqual(post3.status_code, 200) - - - def test_POSTBIN_GET_POST_FILES_WITH_PARAMS(self): - - for service in SERVICES: - - url = service('post') - post = requests.post(url, - files={'some': open('test_requests.py')}, - data={'some': 'data'}) - - self.assertEqual(post.status_code, 200) - - - def test_POSTBIN_GET_POST_FILES_WITH_HEADERS(self): - - for service in SERVICES: - - url = service('post') - - post2 = requests.post(url, - files={'some': open('test_requests.py')}, - headers = {'User-Agent': 'requests-tests'}) - - self.assertEqual(post2.status_code, 200) - - - def test_nonzero_evaluation(self): - - for service in SERVICES: - - r = requests.get(service('status', '500')) - self.assertEqual(bool(r), False) - - r = requests.get(service('/')) - self.assertEqual(bool(r), True) - - - def test_request_ok_set(self): - - for service in SERVICES: - - r = requests.get(service('status', '404')) - self.assertEqual(r.ok, False) - - - def test_status_raising(self): - r = requests.get(httpbin('status', '404')) - self.assertRaises(requests.HTTPError, r.raise_for_status) - - r = requests.get(httpbin('status', '200')) - self.assertFalse(r.error) - r.raise_for_status() - - - def test_cookie_jar(self): - - jar = cookielib.CookieJar() - self.assertFalse(jar) - - url = httpbin('cookies', 'set', 'requests_cookie', 'awesome') - r = requests.get(url, cookies=jar) - self.assertTrue(jar) - - cookie_found = False - for cookie in jar: - if cookie.name == 'requests_cookie': - self.assertEquals(cookie.value, 'awesome') - cookie_found = True - self.assertTrue(cookie_found) - - r = requests.get(httpbin('cookies'), cookies=jar) - self.assertTrue('awesome' in r.content) - - - def test_decompress_gzip(self): - - r = requests.get(httpbin('gzip')) - r.content.decode('ascii') - - - def test_unicode_get(self): - - for service in SERVICES: - - url = service('/') - - requests.get(url, params={'foo': u'føø'}) - requests.get(url, params={u'føø': u'føø'}) - requests.get(url, params={'føø': 'føø'}) - requests.get(url, params={'foo': u'foo'}) - requests.get(service('ø'), params={'foo': u'foo'}) - - - def test_httpauth_recursion(self): - - http_auth = ('user', 'BADpass') - - for service in SERVICES: - r = requests.get(service('basic-auth', 'user', 'pass'), auth=http_auth) - self.assertEquals(r.status_code, 401) - - - def test_settings(self): - - def test(): - r = requests.get(httpbin('')) - r.raise_for_status() - - with requests.settings(timeout=0.0000001): - self.assertRaises(requests.Timeout, test) - - with requests.settings(timeout=100): - requests.get(httpbin('')) - - - def test_urlencoded_post_data(self): - - for service in SERVICES: - - r = requests.post(service('post'), data=dict(test='fooaowpeuf')) - - self.assertEquals(r.status_code, 200) - self.assertEquals(r.headers['content-type'], 'application/json') - self.assertEquals(r.url, service('post')) - - rbody = json.loads(r.content) - - self.assertEquals(rbody.get('form'), dict(test='fooaowpeuf')) - self.assertEquals(rbody.get('data'), '') - - - def test_nonurlencoded_post_data(self): - - for service in SERVICES: - - r = requests.post(service('post'), data='fooaowpeuf') - - self.assertEquals(r.status_code, 200) - self.assertEquals(r.headers['content-type'], 'application/json') - self.assertEquals(r.url, service('post')) - - rbody = json.loads(r.content) - # Body wasn't valid url encoded data, so the server returns None as - # "form" and the raw body as "data". - self.assertEquals(rbody.get('form'), None) - self.assertEquals(rbody.get('data'), 'fooaowpeuf') - - - def test_urlencoded_post_querystring(self): - - for service in SERVICES: - - r = requests.post(service('post'), params=dict(test='fooaowpeuf')) - - self.assertEquals(r.status_code, 200) - self.assertEquals(r.headers['content-type'], 'application/json') - self.assertEquals(r.url, service('post?test=fooaowpeuf')) - - rbody = json.loads(r.content) - self.assertEquals(rbody.get('form'), {}) # No form supplied - self.assertEquals(rbody.get('data'), '') - - - def test_nonurlencoded_post_querystring(self): - - for service in SERVICES: - - r = requests.post(service('post'), params='fooaowpeuf') - - self.assertEquals(r.status_code, 200) - self.assertEquals(r.headers['content-type'], 'application/json') - self.assertEquals(r.url, service('post?fooaowpeuf')) - - rbody = json.loads(r.content) - self.assertEquals(rbody.get('form'), {}) # No form supplied - self.assertEquals(rbody.get('data'), '') - - - def test_urlencoded_post_query_and_data(self): - - for service in SERVICES: - - r = requests.post( - service('post'), - params=dict(test='fooaowpeuf'), - data=dict(test2="foobar")) - - self.assertEquals(r.status_code, 200) - self.assertEquals(r.headers['content-type'], 'application/json') - self.assertEquals(r.url, service('post?test=fooaowpeuf')) - - rbody = json.loads(r.content) - self.assertEquals(rbody.get('form'), dict(test2='foobar')) - self.assertEquals(rbody.get('data'), '') - - - def test_nonurlencoded_post_query_and_data(self): - - for service in SERVICES: - - r = requests.post(service('post'), - params='fooaowpeuf', data="foobar") - - self.assertEquals(r.status_code, 200) - self.assertEquals(r.headers['content-type'], 'application/json') - self.assertEquals(r.url, service('post?fooaowpeuf')) - - rbody = json.loads(r.content) - - self.assertEquals(rbody.get('form'), None) - self.assertEquals(rbody.get('data'), 'foobar') - - - def test_idna(self): - r = requests.get(u'http://➡.ws/httpbin') - assert 'httpbin' in r.url - - - def test_urlencoded_get_query_multivalued_param(self): - - for service in SERVICES: - - r = requests.get(service('get'), params=dict(test=['foo','baz'])) - self.assertEquals(r.status_code, 200) - self.assertEquals(r.url, service('get?test=foo&test=baz')) - - - def test_urlencoded_post_querystring_multivalued(self): - - for service in SERVICES: - - r = requests.post(service('post'), params=dict(test=['foo','baz'])) - self.assertEquals(r.status_code, 200) - self.assertEquals(r.headers['content-type'], 'application/json') - self.assertEquals(r.url, service('post?test=foo&test=baz')) - - rbody = json.loads(r.content) - self.assertEquals(rbody.get('form'), {}) # No form supplied - self.assertEquals(rbody.get('data'), '') - - - def test_urlencoded_post_query_multivalued_and_data(self): - - for service in SERVICES: - - r = requests.post( - service('post'), - params=dict(test=['foo','baz']), - data=dict(test2="foobar",test3=['foo','baz'])) - - self.assertEquals(r.status_code, 200) - self.assertEquals(r.headers['content-type'], 'application/json') - self.assertEquals(r.url, service('post?test=foo&test=baz')) - rbody = json.loads(r.content) - self.assertEquals(rbody.get('form'), dict(test2='foobar',test3='foo')) - self.assertEquals(rbody.get('data'), '') - - - def test_redirect_history(self): - - for service in SERVICES: - - r = requests.get(service('redirect', '3')) - self.assertEquals(r.status_code, 200) - self.assertEquals(len(r.history), 3) - - - def test_relative_redirect_history(self): - - for service in SERVICES: - - r = requests.get(service('relative-redirect', '3')) - self.assertEquals(r.status_code, 200) - self.assertEquals(len(r.history), 3) - - - def test_session_HTTP_200_OK_GET(self): - - s = Session() - r = s.get(httpbin('/')) - self.assertEqual(r.status_code, 200) - - - def test_session_HTTPS_200_OK_GET(self): - - s = Session() - r = s.get(httpsbin('/')) - self.assertEqual(r.status_code, 200) - - - def test_session_persistent_headers(self): - - heads = {'User-agent': 'Mozilla/5.0'} - - s = Session() - s.headers = heads - # Make 2 requests from Session object, should send header both times - r1 = s.get(httpbin('user-agent')) - - assert heads['User-agent'] in r1.content - r2 = s.get(httpbin('user-agent')) - - assert heads['User-agent'] in r2.content - self.assertEqual(r2.status_code, 200) - - - - def test_curl_HTTP_OK_GET(self): - curl_str = 'curl -L -X GET -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" "http://httpbin.org//"' - r = requests.get(httpbin('/')) - self.assertEqual(curl_from_request(r.request), curl_str) - - - def test_curl_HTTP_OK_GET_WITH_PARAMS(self): - curl_str = 'curl -L -X GET -H "Accept-Encoding:gzip" -H "User-agent:Mozilla/5.0" "http://httpbin.org/user-agent"' - - heads = {'User-agent': 'Mozilla/5.0'} - r = requests.get(httpbin('user-agent'), headers=heads) - self.assertEqual(curl_from_request(r.request), curl_str) - - - def test_curl_HTTP_OK_HEAD(self): - curl_str ='curl -L -I -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" "http://httpbin.org//"' - r = requests.head(httpbin('/')) - self.assertEqual(curl_from_request(r.request), curl_str) - - - def test_curl_HTTP_OK_PATCH(self): - curl_str = 'curl -L -X PATCH -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" "http://httpbin.org/patch"' - r = requests.patch(httpbin('patch')) - self.assertEqual(curl_from_request(r.request), curl_str) - - - def test_curl_AUTH_HTTPS_OK_GET(self): - curl_str = 'curl -L -u "user:pass" -X GET -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" "https://httpbin.ep.io/basic-auth/user/pass"' - auth = ('user', 'pass') - r = requests.get(httpsbin('basic-auth', 'user', 'pass'), auth=auth) - self.assertEqual(curl_from_request(r.request), curl_str) - - - def test_curl_POSTBIN_GET_POST_FILES(self): - curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -d "some=data" "http://httpbin.org/post"' - post = requests.post(httpbin('post'), data={'some': 'data'}) - self.assertEqual(curl_from_request(post.request), curl_str) - - curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -F "some=@test_requests.py" "https://httpbin.ep.io/post"' - post2 = requests.post(httpsbin('post'), files={'some': open('test_requests.py')}) - self.assertEqual(curl_from_request(post2.request), curl_str) - - curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -d \'[{"some": "json"}]\' "http://httpbin.org/post"' - post3 = requests.post(httpbin('post'), data='[{"some": "json"}]') - self.assertEqual(curl_from_request(post3.request), curl_str) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/unit/test_requests_api.py b/tests/unit/test_requests_api.py deleted file mode 100755 index ae63ab85..00000000 --- a/tests/unit/test_requests_api.py +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import unittest -import mock -import sys -import os -sys.path.append(os.getcwd()) - -try: - import omnijson as json -except ImportError: - import json - -import requests -from requests.models import Response - -class RequestsAPIUnitTests(unittest.TestCase): - """Requests API unit test cases.""" - - def setUp(self): - pass - - - def tearDown(self): - """Teardown.""" - pass - - - @mock.patch('requests.api.dispatch_hook') - @mock.patch('requests.api.Request') - @mock.patch('requests.api.cookiejar_from_dict') - def test_request(self, mock_cjar, mock_request, mock_hook): - args = dict( - method = None, - url = None, - data = None, - params = None, - headers = None, - cookiejar = None, - files = None, - auth = None, - timeout = 1, - allow_redirects = None, - proxies = None, - ) - hooks = {'args': args, 'pre_request': mock_request, - 'post_request': mock_request, 'response': 'response'} - sideeffect = lambda x,y,z: hooks[x] - mock_cjar.return_value = None - mock_request.send = mock.Mock(return_value={}) - mock_request.response = "response" - mock_hook.side_effect = sideeffect - - r = requests.request('get','http://google.com') - - - mock_cjar.assert_called_once_with({}) - mock_hook.assert_called__with('args', None, args) - mock_request.assert_called_once_with(**args) - mock_hook.assert_called__with('pre_request', None, mock_request) - mock_request.send.assert_called_once_with() - mock_hook.assert_called__with('post_request', None, mock_request) - mock_hook.assert_called__with('response', None, mock_request) - self.assertEqual(r, "response") - - - - @mock.patch('requests.api.request') - def test_http_get(self, mock_request): - mock_request.return_value = Response() - requests.get('http://google.com') - mock_request.assert_called_once_with('get', 'http://google.com', - allow_redirects= True) - - @mock.patch('requests.api.request') - def test_http_get_with_kwargs(self, mock_request): - mock_request.return_value = Response() - requests.get('http://google.com', - params="params", data="data", headers="headers", - cookies="cookies", - files="files", auth="auth", timeout="timeout", - allow_redirects=False, - proxies="proxies", hooks="hooks") - mock_request.assert_called_once_with('get', 'http://google.com', - params="params", data="data", headers="headers", - cookies="cookies", - files="files", auth="auth", timeout="timeout", - allow_redirects=False, - proxies="proxies", hooks="hooks") - - @mock.patch('requests.api.request') - def test_http_head(self, mock_request): - mock_request.return_value = Response() - requests.head('http://google.com') - mock_request.assert_called_once_with('head', 'http://google.com', - allow_redirects= True) - - @mock.patch('requests.api.request') - def test_http_head_with_kwargs(self, mock_request): - mock_request.return_value = Response() - requests.head('http://google.com', - params="params", data="data", headers="headers", - cookies="cookies", - files="files", auth="auth", timeout="timeout", - allow_redirects=False, - proxies="proxies", hooks="hooks") - mock_request.assert_called_once_with('head', 'http://google.com', - params="params", data="data", headers="headers", - cookies="cookies", - files="files", auth="auth", timeout="timeout", - allow_redirects=False, - proxies="proxies", hooks="hooks") - - @mock.patch('requests.api.request') - def test_http_post(self, mock_request): - mock_request.return_value = Response() - requests.post('http://google.com', {}) - mock_request.assert_called_once_with('post', 'http://google.com', - data= {}) - - @mock.patch('requests.api.request') - def test_http_post_with_kwargs(self, mock_request): - mock_request.return_value = Response() - requests.post('http://google.com', - params="params", data="data", headers="headers", - cookies="cookies", - files="files", auth="auth", timeout="timeout", - allow_redirects=False, - proxies="proxies", hooks="hooks") - mock_request.assert_called_once_with('post', 'http://google.com', - params="params", data="data", headers="headers", - cookies="cookies", - files="files", auth="auth", timeout="timeout", - allow_redirects=False, - proxies="proxies", hooks="hooks") - - - @mock.patch('requests.api.request') - def test_http_put(self, mock_request): - mock_request.return_value = Response() - requests.put('http://google.com', {}) - mock_request.assert_called_once_with('put', 'http://google.com', - data= {}) - - @mock.patch('requests.api.request') - def test_http_put_with_kwargs(self, mock_request): - mock_request.return_value = Response() - requests.put('http://google.com', - params="params", data="data", headers="headers", - cookies="cookies", - files="files", auth="auth", timeout="timeout", - allow_redirects=False, - proxies="proxies", hooks="hooks") - mock_request.assert_called_once_with('put', 'http://google.com', - params="params", data="data", headers="headers", - cookies="cookies", - files="files", auth="auth", timeout="timeout", - allow_redirects=False, - proxies="proxies", hooks="hooks") - - - @mock.patch('requests.api.request') - def test_http_patch(self, mock_request): - mock_request.return_value = Response() - requests.patch('http://google.com', {}) - mock_request.assert_called_once_with('patch', 'http://google.com', - data= {}) - - @mock.patch('requests.api.request') - def test_http_patch_with_kwargs(self, mock_request): - mock_request.return_value = Response() - requests.patch('http://google.com', - params="params", data="data", headers="headers", - cookies="cookies", - files="files", auth="auth", timeout="timeout", - allow_redirects=False, - proxies="proxies", hooks="hooks") - mock_request.assert_called_once_with('patch', 'http://google.com', - params="params", data="data", headers="headers", - cookies="cookies", - files="files", auth="auth", timeout="timeout", - allow_redirects=False, - proxies="proxies", hooks="hooks") - - @mock.patch('requests.api.request') - def test_http_delete(self, mock_request): - mock_request.return_value = Response() - requests.delete('http://google.com') - mock_request.assert_called_once_with('delete', 'http://google.com') - - @mock.patch('requests.api.request') - def test_http_delete_with_kwargs(self, mock_request): - mock_request.return_value = Response() - requests.delete('http://google.com', - params="params", data="data", headers="headers", - cookies="cookies", - files="files", auth="auth", timeout="timeout", - allow_redirects=False, - proxies="proxies", hooks="hooks") - mock_request.assert_called_once_with('delete', 'http://google.com', - params="params", data="data", headers="headers", - cookies="cookies", - files="files", auth="auth", timeout="timeout", - allow_redirects=False, - proxies="proxies", hooks="hooks") - - -if __name__ == '__main__': - unittest.main() From 2361a6fc79f3af84550478cd43e0f9ab26e714f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Mon, 3 Oct 2011 11:16:43 +0200 Subject: [PATCH 130/255] Renamed get_clean_url -> cleanup_url. --- requests/models.py | 2 +- requests/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index a47a16b6..2ac8c33a 100644 --- a/requests/models.py +++ b/requests/models.py @@ -199,7 +199,7 @@ class Request(object): # Add the old request to the history collector. history.append(r) - url = get_clean_url(r.headers['location'], parent_url=self.url) + url = cleanup_url(r.headers['location'], parent_url=self.url) # If 303, convert to idempotent GET. # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 diff --git a/requests/utils.py b/requests/utils.py index 1dcf3c2f..0a114768 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -44,7 +44,7 @@ def encode_params(params): else: return params -def get_clean_url(url, parent_url=None): +def cleanup_url(url, parent_url=None): # Handle redirection without scheme (see: RFC 1808 Section 4) if url.startswith('//'): parsed_rurl = urlparse(parent_url) @@ -73,7 +73,7 @@ def get_clean_url(url, parent_url=None): def build_url(url, query_params): """Build the actual URL to use.""" - url = get_clean_url(url) + url = cleanup_url(url) query_params = encode_params(query_params) From 04bcb74970c49ea6bda3881cb02beed2e3359b7b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 4 Oct 2011 11:33:11 -0400 Subject: [PATCH 131/255] Add a trailing slash for twitter.com --- requests/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requests/utils.py b/requests/utils.py index 0a114768..832a11f3 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -54,6 +54,10 @@ def cleanup_url(url, parent_url=None): if netloc: netloc = netloc.encode('idna') + # Add a trailing slash to root domain reqests. + if not len(path): + path = '/' + if isinstance(path, unicode): path = path.encode('utf-8') From 608d05706696b292a6af021954cfbc2515a102fa Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 4 Oct 2011 11:33:19 -0400 Subject: [PATCH 132/255] nice and fast nose tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6ec49602..12ab7fc3 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ init: pip install -r reqs.txt test: - python test_requests.py + nosetests tests/integration_tests.py --processes=25 site: cd docs; make dirhtml From b970d468d1251c51d28b0ec142a12f817cf19925 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 6 Oct 2011 11:58:03 -0400 Subject: [PATCH 133/255] whoops --- requests/packages/urllib3/poolmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index acf1c5a1..1d757d19 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -72,7 +72,7 @@ class PoolManager(object): port = port or port_by_scheme.get(scheme, 80) r = self.connection_from_host(host, port=port, scheme=scheme) - print r.__dict__ + return r def urlopen(self, method, url, **kw): From a8695ecd32f18135ae5f34f466382993a4cb7d3d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 9 Oct 2011 07:11:48 -0400 Subject: [PATCH 134/255] v0.6.2 --- HISTORY.rst | 5 +++++ requests/core.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b09392b5..fbd4a726 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,11 @@ History ------- +0.6.2 (2011-10-09) +++++++++++++++++++ + +* GET/HEAD obeys follow_redirect=False + 0.6.1 (2011-08-20) ++++++++++++++++++ diff --git a/requests/core.py b/requests/core.py index 8ba34a2f..505f8a24 100644 --- a/requests/core.py +++ b/requests/core.py @@ -12,8 +12,8 @@ This module implements the main Requests system. """ __title__ = 'requests' -__version__ = '0.6.1' -__build__ = 0x000601 +__version__ = '0.6.2' +__build__ = 0x000602 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' From 78b5d9ed45d3f969e38e6772501da446db1efd59 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 9 Oct 2011 08:25:17 -0400 Subject: [PATCH 135/255] Fix accidental-merge docstrings --- requests/models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 2ac8c33a..98cc4b30 100644 --- a/requests/models.py +++ b/requests/models.py @@ -4,6 +4,7 @@ requests.models ~~~~~~~~~~~~~~~ +This module contains the primary classes that power Requests. """ import urllib @@ -243,7 +244,10 @@ class Request(object): def send(self, anyway=False): - """Sends the shit.""" + """Sends the HTTP Request. Populates `Request.response`. + + Returns True if everything went according to plan. + """ # Safety check. self._checks() From 03babafc7b16bd84b13f405fd66ccb1751f4aaee Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 9 Oct 2011 08:34:13 -0400 Subject: [PATCH 136/255] Weakrefs for Reuqest/Response. Closes #186 --- requests/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 98cc4b30..78290570 100644 --- a/requests/models.py +++ b/requests/models.py @@ -11,6 +11,7 @@ import urllib import zlib from Cookie import SimpleCookie from urlparse import urlparse, urlunparse, urljoin +from weakref import ref from .packages import urllib3 from .packages.urllib3.filepost import encode_multipart_formdata @@ -240,7 +241,8 @@ class Request(object): self.response = r # Give Response some context. - self.response.request = self + self.response.request = ref(self)() + self.response.request.response = ref(self.response)() def send(self, anyway=False): From 9697290111d0a6c664f97a2dd452a9132ca2c384 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 9 Oct 2011 16:04:00 -0400 Subject: [PATCH 137/255] added nose to reqs --- reqs.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/reqs.txt b/reqs.txt index 23ee03b2..edacdc81 100644 --- a/reqs.txt +++ b/reqs.txt @@ -1 +1,2 @@ +nose mock \ No newline at end of file From e9f94e698d3cb1e84d383dbc5a7393e55140b542 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 10 Oct 2011 01:27:05 -0400 Subject: [PATCH 138/255] lost in merge --- tests/api_tests.py | 209 +++++++++++++++ tests/integration_tests.py | 528 +++++++++++++++++++++++++++++++++++++ 2 files changed, 737 insertions(+) create mode 100755 tests/api_tests.py create mode 100755 tests/integration_tests.py diff --git a/tests/api_tests.py b/tests/api_tests.py new file mode 100755 index 00000000..ae63ab85 --- /dev/null +++ b/tests/api_tests.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import unittest +import mock +import sys +import os +sys.path.append(os.getcwd()) + +try: + import omnijson as json +except ImportError: + import json + +import requests +from requests.models import Response + +class RequestsAPIUnitTests(unittest.TestCase): + """Requests API unit test cases.""" + + def setUp(self): + pass + + + def tearDown(self): + """Teardown.""" + pass + + + @mock.patch('requests.api.dispatch_hook') + @mock.patch('requests.api.Request') + @mock.patch('requests.api.cookiejar_from_dict') + def test_request(self, mock_cjar, mock_request, mock_hook): + args = dict( + method = None, + url = None, + data = None, + params = None, + headers = None, + cookiejar = None, + files = None, + auth = None, + timeout = 1, + allow_redirects = None, + proxies = None, + ) + hooks = {'args': args, 'pre_request': mock_request, + 'post_request': mock_request, 'response': 'response'} + sideeffect = lambda x,y,z: hooks[x] + mock_cjar.return_value = None + mock_request.send = mock.Mock(return_value={}) + mock_request.response = "response" + mock_hook.side_effect = sideeffect + + r = requests.request('get','http://google.com') + + + mock_cjar.assert_called_once_with({}) + mock_hook.assert_called__with('args', None, args) + mock_request.assert_called_once_with(**args) + mock_hook.assert_called__with('pre_request', None, mock_request) + mock_request.send.assert_called_once_with() + mock_hook.assert_called__with('post_request', None, mock_request) + mock_hook.assert_called__with('response', None, mock_request) + self.assertEqual(r, "response") + + + + @mock.patch('requests.api.request') + def test_http_get(self, mock_request): + mock_request.return_value = Response() + requests.get('http://google.com') + mock_request.assert_called_once_with('get', 'http://google.com', + allow_redirects= True) + + @mock.patch('requests.api.request') + def test_http_get_with_kwargs(self, mock_request): + mock_request.return_value = Response() + requests.get('http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + mock_request.assert_called_once_with('get', 'http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + + @mock.patch('requests.api.request') + def test_http_head(self, mock_request): + mock_request.return_value = Response() + requests.head('http://google.com') + mock_request.assert_called_once_with('head', 'http://google.com', + allow_redirects= True) + + @mock.patch('requests.api.request') + def test_http_head_with_kwargs(self, mock_request): + mock_request.return_value = Response() + requests.head('http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + mock_request.assert_called_once_with('head', 'http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + + @mock.patch('requests.api.request') + def test_http_post(self, mock_request): + mock_request.return_value = Response() + requests.post('http://google.com', {}) + mock_request.assert_called_once_with('post', 'http://google.com', + data= {}) + + @mock.patch('requests.api.request') + def test_http_post_with_kwargs(self, mock_request): + mock_request.return_value = Response() + requests.post('http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + mock_request.assert_called_once_with('post', 'http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + + + @mock.patch('requests.api.request') + def test_http_put(self, mock_request): + mock_request.return_value = Response() + requests.put('http://google.com', {}) + mock_request.assert_called_once_with('put', 'http://google.com', + data= {}) + + @mock.patch('requests.api.request') + def test_http_put_with_kwargs(self, mock_request): + mock_request.return_value = Response() + requests.put('http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + mock_request.assert_called_once_with('put', 'http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + + + @mock.patch('requests.api.request') + def test_http_patch(self, mock_request): + mock_request.return_value = Response() + requests.patch('http://google.com', {}) + mock_request.assert_called_once_with('patch', 'http://google.com', + data= {}) + + @mock.patch('requests.api.request') + def test_http_patch_with_kwargs(self, mock_request): + mock_request.return_value = Response() + requests.patch('http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + mock_request.assert_called_once_with('patch', 'http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + + @mock.patch('requests.api.request') + def test_http_delete(self, mock_request): + mock_request.return_value = Response() + requests.delete('http://google.com') + mock_request.assert_called_once_with('delete', 'http://google.com') + + @mock.patch('requests.api.request') + def test_http_delete_with_kwargs(self, mock_request): + mock_request.return_value = Response() + requests.delete('http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + mock_request.assert_called_once_with('delete', 'http://google.com', + params="params", data="data", headers="headers", + cookies="cookies", + files="files", auth="auth", timeout="timeout", + allow_redirects=False, + proxies="proxies", hooks="hooks") + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/integration_tests.py b/tests/integration_tests.py new file mode 100755 index 00000000..4cfb627d --- /dev/null +++ b/tests/integration_tests.py @@ -0,0 +1,528 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from __future__ import with_statement + +import unittest +import cookielib + +try: + import omnijson as json +except ImportError: + import json + +import requests + +from requests.sessions import Session +from requests.utils import curl_from_request + + +HTTPBIN_URL = 'http://httpbin.org/' +HTTPSBIN_URL = 'https://httpbin.herokuapp.com/' + +# HTTPBIN_URL = 'http://staging.httpbin.org/' +# HTTPSBIN_URL = 'https://httpbin-staging.ep.io/' + + +def httpbin(*suffix): + """Returns url for HTTPBIN resource.""" + + return HTTPBIN_URL + '/'.join(suffix) + + +def httpsbin(*suffix): + """Returns url for HTTPSBIN resource.""" + + return HTTPSBIN_URL + '/'.join(suffix) + + +SERVICES = (httpbin, httpsbin) + + + +class RequestsTestSuite(unittest.TestCase): + """Requests test cases.""" + + # It goes to eleven. + _multiprocess_can_split_ = True + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_invalid_url(self): + self.assertRaises(ValueError, requests.get, 'hiwpefhipowhefopw') + + def test_HTTP_200_OK_GET(self): + r = requests.get(httpbin('/')) + self.assertEqual(r.status_code, 200) + + def test_HTTP_302_ALLOW_REDIRECT_GET(self): + r = requests.get(httpbin('redirect', '1')) + self.assertEqual(r.status_code, 200) + + def test_HTTP_302_GET(self): + r = requests.get(httpbin('redirect', '1'), allow_redirects=False) + self.assertEqual(r.status_code, 302) + + def test_HTTPS_200_OK_GET(self): + r = requests.get(httpsbin('/')) + self.assertEqual(r.status_code, 200) + + + def test_HTTP_200_OK_GET_WITH_PARAMS(self): + heads = {'User-agent': 'Mozilla/5.0'} + + r = requests.get(httpbin('user-agent'), headers=heads) + + assert heads['User-agent'] in r.content + self.assertEqual(r.status_code, 200) + + + def test_HTTP_200_OK_GET_WITH_MIXED_PARAMS(self): + heads = {'User-agent': 'Mozilla/5.0'} + + r = requests.get(httpbin('get') + '?test=true', params={'q': 'test'}, headers=heads) + self.assertEqual(r.status_code, 200) + + + def test_user_agent_transfers(self): + """Issue XX""" + + heads = { + 'User-agent': + 'Mozilla/5.0 (github.com/kennethreitz/requests)' + } + + r = requests.get(httpbin('user-agent'), headers=heads); + self.assertTrue(heads['User-agent'] in r.content) + + heads = { + 'user-agent': + 'Mozilla/5.0 (github.com/kennethreitz/requests)' + } + + r = requests.get(httpbin('user-agent'), headers=heads); + self.assertTrue(heads['user-agent'] in r.content) + + + def test_HTTP_200_OK_HEAD(self): + r = requests.head(httpbin('/')) + self.assertEqual(r.status_code, 200) + + + def test_HTTPS_200_OK_HEAD(self): + r = requests.head(httpsbin('/')) + self.assertEqual(r.status_code, 200) + + + def test_HTTP_200_OK_PUT(self): + r = requests.put(httpbin('put')) + self.assertEqual(r.status_code, 200) + + + def test_HTTPS_200_OK_PUT(self): + r = requests.put(httpsbin('put')) + self.assertEqual(r.status_code, 200) + + + def test_HTTP_200_OK_PATCH(self): + r = requests.patch(httpbin('patch')) + self.assertEqual(r.status_code, 200) + + + def test_HTTPS_200_OK_PATCH(self): + r = requests.patch(httpsbin('patch')) + self.assertEqual(r.status_code, 200) + + + def test_AUTH_HTTP_200_OK_GET(self): + + for service in SERVICES: + + auth = ('user', 'pass') + url = service('basic-auth', 'user', 'pass') + + r = requests.get(url, auth=auth) + # print r.__dict__ + self.assertEqual(r.status_code, 200) + + + r = requests.get(url) + self.assertEqual(r.status_code, 200) + + + def test_POSTBIN_GET_POST_FILES(self): + + for service in SERVICES: + + url = service('post') + post = requests.post(url).raise_for_status() + + post = requests.post(url, data={'some': 'data'}) + self.assertEqual(post.status_code, 200) + + post2 = requests.post(url, files={'some': open('test_requests.py')}) + self.assertEqual(post2.status_code, 200) + + post3 = requests.post(url, data='[{"some": "json"}]') + self.assertEqual(post3.status_code, 200) + + + def test_POSTBIN_GET_POST_FILES_WITH_PARAMS(self): + + for service in SERVICES: + + url = service('post') + post = requests.post(url, + files={'some': open('test_requests.py')}, + data={'some': 'data'}) + + self.assertEqual(post.status_code, 200) + + + def test_POSTBIN_GET_POST_FILES_WITH_HEADERS(self): + + for service in SERVICES: + + url = service('post') + + post2 = requests.post(url, + files={'some': open('test_requests.py')}, + headers = {'User-Agent': 'requests-tests'}) + + self.assertEqual(post2.status_code, 200) + + + def test_nonzero_evaluation(self): + + for service in SERVICES: + + r = requests.get(service('status', '500')) + self.assertEqual(bool(r), False) + + r = requests.get(service('/')) + self.assertEqual(bool(r), True) + + + def test_request_ok_set(self): + + for service in SERVICES: + + r = requests.get(service('status', '404')) + self.assertEqual(r.ok, False) + + + def test_status_raising(self): + r = requests.get(httpbin('status', '404')) + self.assertRaises(requests.HTTPError, r.raise_for_status) + + r = requests.get(httpbin('status', '200')) + self.assertFalse(r.error) + r.raise_for_status() + + + def test_cookie_jar(self): + + jar = cookielib.CookieJar() + self.assertFalse(jar) + + url = httpbin('cookies', 'set', 'requests_cookie', 'awesome') + r = requests.get(url, cookies=jar) + self.assertTrue(jar) + + cookie_found = False + for cookie in jar: + if cookie.name == 'requests_cookie': + self.assertEquals(cookie.value, 'awesome') + cookie_found = True + self.assertTrue(cookie_found) + + r = requests.get(httpbin('cookies'), cookies=jar) + self.assertTrue('awesome' in r.content) + + + def test_decompress_gzip(self): + + r = requests.get(httpbin('gzip')) + r.content.decode('ascii') + + + def test_unicode_get(self): + + for service in SERVICES: + + url = service('/') + + requests.get(url, params={'foo': u'føø'}) + requests.get(url, params={u'føø': u'føø'}) + requests.get(url, params={'føø': 'føø'}) + requests.get(url, params={'foo': u'foo'}) + requests.get(service('ø'), params={'foo': u'foo'}) + + + def test_httpauth_recursion(self): + + http_auth = ('user', 'BADpass') + + for service in SERVICES: + r = requests.get(service('basic-auth', 'user', 'pass'), auth=http_auth) + self.assertEquals(r.status_code, 401) + + + def test_settings(self): + + def test(): + r = requests.get(httpbin('')) + r.raise_for_status() + + with requests.settings(timeout=0.0000001): + self.assertRaises(requests.Timeout, test) + + with requests.settings(timeout=100): + requests.get(httpbin('')) + + + def test_urlencoded_post_data(self): + + for service in SERVICES: + + r = requests.post(service('post'), data=dict(test='fooaowpeuf')) + + self.assertEquals(r.status_code, 200) + self.assertEquals(r.headers['content-type'], 'application/json') + self.assertEquals(r.url, service('post')) + + rbody = json.loads(r.content) + + self.assertEquals(rbody.get('form'), dict(test='fooaowpeuf')) + self.assertEquals(rbody.get('data'), '') + + + def test_nonurlencoded_post_data(self): + + for service in SERVICES: + + r = requests.post(service('post'), data='fooaowpeuf') + + self.assertEquals(r.status_code, 200) + self.assertEquals(r.headers['content-type'], 'application/json') + self.assertEquals(r.url, service('post')) + + rbody = json.loads(r.content) + # Body wasn't valid url encoded data, so the server returns None as + # "form" and the raw body as "data". + self.assertEquals(rbody.get('form'), None) + self.assertEquals(rbody.get('data'), 'fooaowpeuf') + + + def test_urlencoded_post_querystring(self): + + for service in SERVICES: + + r = requests.post(service('post'), params=dict(test='fooaowpeuf')) + + self.assertEquals(r.status_code, 200) + self.assertEquals(r.headers['content-type'], 'application/json') + self.assertEquals(r.url, service('post?test=fooaowpeuf')) + + rbody = json.loads(r.content) + self.assertEquals(rbody.get('form'), {}) # No form supplied + self.assertEquals(rbody.get('data'), '') + + + def test_nonurlencoded_post_querystring(self): + + for service in SERVICES: + + r = requests.post(service('post'), params='fooaowpeuf') + + self.assertEquals(r.status_code, 200) + self.assertEquals(r.headers['content-type'], 'application/json') + self.assertEquals(r.url, service('post?fooaowpeuf')) + + rbody = json.loads(r.content) + self.assertEquals(rbody.get('form'), {}) # No form supplied + self.assertEquals(rbody.get('data'), '') + + + def test_urlencoded_post_query_and_data(self): + + for service in SERVICES: + + r = requests.post( + service('post'), + params=dict(test='fooaowpeuf'), + data=dict(test2="foobar")) + + self.assertEquals(r.status_code, 200) + self.assertEquals(r.headers['content-type'], 'application/json') + self.assertEquals(r.url, service('post?test=fooaowpeuf')) + + rbody = json.loads(r.content) + self.assertEquals(rbody.get('form'), dict(test2='foobar')) + self.assertEquals(rbody.get('data'), '') + + + def test_nonurlencoded_post_query_and_data(self): + + for service in SERVICES: + + r = requests.post(service('post'), + params='fooaowpeuf', data="foobar") + + self.assertEquals(r.status_code, 200) + self.assertEquals(r.headers['content-type'], 'application/json') + self.assertEquals(r.url, service('post?fooaowpeuf')) + + rbody = json.loads(r.content) + + self.assertEquals(rbody.get('form'), None) + self.assertEquals(rbody.get('data'), 'foobar') + + + def test_idna(self): + r = requests.get(u'http://➡.ws/httpbin') + assert 'httpbin' in r.url + + + def test_urlencoded_get_query_multivalued_param(self): + + for service in SERVICES: + + r = requests.get(service('get'), params=dict(test=['foo','baz'])) + self.assertEquals(r.status_code, 200) + self.assertEquals(r.url, service('get?test=foo&test=baz')) + + + def test_urlencoded_post_querystring_multivalued(self): + + for service in SERVICES: + + r = requests.post(service('post'), params=dict(test=['foo','baz'])) + self.assertEquals(r.status_code, 200) + self.assertEquals(r.headers['content-type'], 'application/json') + self.assertEquals(r.url, service('post?test=foo&test=baz')) + + rbody = json.loads(r.content) + self.assertEquals(rbody.get('form'), {}) # No form supplied + self.assertEquals(rbody.get('data'), '') + + + def test_urlencoded_post_query_multivalued_and_data(self): + + for service in SERVICES: + + r = requests.post( + service('post'), + params=dict(test=['foo','baz']), + data=dict(test2="foobar",test3=['foo','baz'])) + + self.assertEquals(r.status_code, 200) + self.assertEquals(r.headers['content-type'], 'application/json') + self.assertEquals(r.url, service('post?test=foo&test=baz')) + rbody = json.loads(r.content) + self.assertEquals(rbody.get('form'), dict(test2='foobar',test3='foo')) + self.assertEquals(rbody.get('data'), '') + + + def test_redirect_history(self): + + for service in SERVICES: + + r = requests.get(service('redirect', '3')) + self.assertEquals(r.status_code, 200) + self.assertEquals(len(r.history), 3) + + + def test_relative_redirect_history(self): + + for service in SERVICES: + + r = requests.get(service('relative-redirect', '3')) + self.assertEquals(r.status_code, 200) + self.assertEquals(len(r.history), 3) + + + def test_session_HTTP_200_OK_GET(self): + + s = Session() + r = s.get(httpbin('/')) + self.assertEqual(r.status_code, 200) + + + def test_session_HTTPS_200_OK_GET(self): + + s = Session() + r = s.get(httpsbin('/')) + self.assertEqual(r.status_code, 200) + + + def test_session_persistent_headers(self): + + heads = {'User-agent': 'Mozilla/5.0'} + + s = Session() + s.headers = heads + # Make 2 requests from Session object, should send header both times + r1 = s.get(httpbin('user-agent')) + + assert heads['User-agent'] in r1.content + r2 = s.get(httpbin('user-agent')) + + assert heads['User-agent'] in r2.content + self.assertEqual(r2.status_code, 200) + + + + def test_curl_HTTP_OK_GET(self): + curl_str = 'curl -L -X GET -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" "http://httpbin.org//"' + r = requests.get(httpbin('/')) + self.assertEqual(curl_from_request(r.request), curl_str) + + + def test_curl_HTTP_OK_GET_WITH_PARAMS(self): + curl_str = 'curl -L -X GET -H "Accept-Encoding:gzip" -H "User-agent:Mozilla/5.0" "http://httpbin.org/user-agent"' + + heads = {'User-agent': 'Mozilla/5.0'} + r = requests.get(httpbin('user-agent'), headers=heads) + self.assertEqual(curl_from_request(r.request), curl_str) + + + def test_curl_HTTP_OK_HEAD(self): + curl_str ='curl -L -I -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" "http://httpbin.org//"' + r = requests.head(httpbin('/')) + self.assertEqual(curl_from_request(r.request), curl_str) + + + def test_curl_HTTP_OK_PATCH(self): + curl_str = 'curl -L -X PATCH -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" "http://httpbin.org/patch"' + r = requests.patch(httpbin('patch')) + self.assertEqual(curl_from_request(r.request), curl_str) + + + def test_curl_AUTH_HTTPS_OK_GET(self): + curl_str = 'curl -L -u "user:pass" -X GET -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" "https://httpbin.ep.io/basic-auth/user/pass"' + auth = ('user', 'pass') + r = requests.get(httpsbin('basic-auth', 'user', 'pass'), auth=auth) + self.assertEqual(curl_from_request(r.request), curl_str) + + + def test_curl_POSTBIN_GET_POST_FILES(self): + curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -d "some=data" "http://httpbin.org/post"' + post = requests.post(httpbin('post'), data={'some': 'data'}) + self.assertEqual(curl_from_request(post.request), curl_str) + + curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -F "some=@test_requests.py" "https://httpbin.ep.io/post"' + post2 = requests.post(httpsbin('post'), files={'some': open('test_requests.py')}) + self.assertEqual(curl_from_request(post2.request), curl_str) + + curl_str = 'curl -L -X POST -H "Accept-Encoding:gzip" -H "User-Agent:python-requests.org" -d \'[{"some": "json"}]\' "http://httpbin.org/post"' + post3 = requests.post(httpbin('post'), data='[{"some": "json"}]') + self.assertEqual(curl_from_request(post3.request), curl_str) + + +if __name__ == '__main__': + unittest.main() From 372e4d94c560473fbff5691e72f8bcc1f5580fb3 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 10 Oct 2011 08:56:23 -0400 Subject: [PATCH 139/255] pyflakes --- reqs.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reqs.txt b/reqs.txt index edacdc81..87d35843 100644 --- a/reqs.txt +++ b/reqs.txt @@ -1,2 +1,3 @@ nose -mock \ No newline at end of file +mock +pyflakes From 7671e3878cbb0f68ed9d1fabb96560227c2df31d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 10 Oct 2011 09:20:24 -0400 Subject: [PATCH 140/255] blah --- Makefile | 4 ++++ reqs.txt | 1 + 2 files changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 12ab7fc3..e30949b1 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,10 @@ init: test: nosetests tests/integration_tests.py --processes=25 +ci: init + nosetests --processes=25 --source-folder=. --with-nosexunit tests/*.py + pyflakes requests | awk -F\: '{printf "%s:%s: [E]%s\n", $1, $2, $3}' > violations.pyflakes.txt + site: cd docs; make dirhtml diff --git a/reqs.txt b/reqs.txt index 87d35843..93bc3911 100644 --- a/reqs.txt +++ b/reqs.txt @@ -1,3 +1,4 @@ nose mock pyflakes +NoseXUnit From 975a183796ac6feccecc8f9d0fd6b4d9eea97e1e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 10 Oct 2011 09:39:44 -0400 Subject: [PATCH 141/255] --core-target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e30949b1..fd4e8ae6 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ test: nosetests tests/integration_tests.py --processes=25 ci: init - nosetests --processes=25 --source-folder=. --with-nosexunit tests/*.py + nosetests --search-test --processes=30 --core-target=$${python} --with-nosexunit tests/*.py pyflakes requests | awk -F\: '{printf "%s:%s: [E]%s\n", $1, $2, $3}' > violations.pyflakes.txt site: From 0cf9ede580e3c661a1fdabdf0f4ce3a484cac363 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 10 Oct 2011 09:58:47 -0400 Subject: [PATCH 142/255] fix builds --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index fd4e8ae6..df91071d 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ test: nosetests tests/integration_tests.py --processes=25 ci: init - nosetests --search-test --processes=30 --core-target=$${python} --with-nosexunit tests/*.py + nosetests --search-test --processes=30 --with-nosexunit tests/*.py pyflakes requests | awk -F\: '{printf "%s:%s: [E]%s\n", $1, $2, $3}' > violations.pyflakes.txt site: From 4ada2e56644e1775e6cf4b2f8ad7c0c02bedff93 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 10 Oct 2011 14:33:35 -0400 Subject: [PATCH 143/255] Added Armin's awesome testimonial \o/ --- docs/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 63ef0669..6ede4bd5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -45,6 +45,10 @@ a U.S. Federal Institution, `Work for Pie `_ use Requests internally. +**Armin Ronacher** + Requests is the perfect example how beautiful an API can be with the + right level of abstraction. + **Daniel Greenfeld** Nuked a 1200 LOC spaghetti code library with 10 lines of code thanks to @kennethreitz's request library. Today has been AWESOME. From 85507dd042362f4e05abc43bcf8862b8ed4c8d5b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 10 Oct 2011 14:36:32 -0400 Subject: [PATCH 144/255] shorten testimonials a bit --- docs/index.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 6ede4bd5..d849ed5f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -60,10 +60,6 @@ use Requests internally. **Rich Leland** Requests is awesome. That is all. -**Steve Pike** - I can never remember how to do it the regular way. - ``import requests; requests.get()`` is just so easy! - User Guide ---------- From 24ffc3610a23ab46a43c0ed5a883a066595e516d Mon Sep 17 00:00:00 2001 From: Pat Nakajima Date: Mon, 10 Oct 2011 15:56:58 -0300 Subject: [PATCH 145/255] just fixed a typo --- docs/community/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/community/faq.rst b/docs/community/faq.rst index b6efc786..5671d3b1 100644 --- a/docs/community/faq.rst +++ b/docs/community/faq.rst @@ -9,7 +9,7 @@ Encoded Data? ------------- Requests automatically decompresses gzip-encoded responses, and does -it's best to decodes response content to unicode when possible. +its best to decodes response content to unicode when possible. You can get direct access to the raw response (and even the socket), if needed as well. From a166dd0ad7772d74410f9b1b0d129b78004c4dac Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 10 Oct 2011 15:58:34 -0300 Subject: [PATCH 146/255] Edited AUTHORS via GitHub --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 0159cdef..1a4a5fb7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -53,3 +53,4 @@ Patches and Suggestions - Mike Waldner - Serge Domkowski - Daniel Miller +- Pat Nakajima From 8d4afcb5b834abe837a0de74fe11c15006fdef43 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 10 Oct 2011 20:07:10 -0400 Subject: [PATCH 147/255] urllib3 upgrade --- requests/packages/urllib3/__init__.py | 24 ++- requests/packages/urllib3/_collections.py | 3 +- requests/packages/urllib3/connectionpool.py | 220 ++++++++++---------- requests/packages/urllib3/filepost.py | 13 ++ requests/packages/urllib3/poolmanager.py | 77 +++++-- requests/packages/urllib3/request.py | 145 +++++++++++++ requests/packages/urllib3/response.py | 19 +- 7 files changed, 357 insertions(+), 144 deletions(-) create mode 100644 requests/packages/urllib3/request.py diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index 5c4b5d17..ae324530 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -8,9 +8,9 @@ urllib3 - Thread-safe connection pooling and re-using. """ -__author__ = "Andrey Petrov (andrey.petrov@shazow.net)" -__license__ = "MIT" -__version__ = "$Rev$" +__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' +__license__ = 'MIT' +__version__ = '1.0' from .connectionpool import ( @@ -27,6 +27,22 @@ from .exceptions import ( SSLError, TimeoutError) -from .poolmanager import PoolManager +from .poolmanager import PoolManager, ProxyManager, proxy_from_url from .response import HTTPResponse from .filepost import encode_multipart_formdata + + +# Set default logging handler to avoid "No handler found" warnings. +import logging +try: + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + +logging.getLogger(__name__).addHandler(NullHandler()) + +# ... Clean up. +del logging +del NullHandler diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py index 2b0de0e8..bd01da33 100644 --- a/requests/packages/urllib3/_collections.py +++ b/requests/packages/urllib3/_collections.py @@ -105,14 +105,13 @@ class RecentlyUsedContainer(MutableMapping): # Discard invalid and excess entries self._prune_entries(len(self._container) - self._maxsize) - def __delitem__(self, key): self._invalidate_entry(key) del self._container[key] del self._access_lookup[key] def __len__(self): - return len(self.access_log) + return self._container.__len__() def __iter__(self): return self._container.__iter__() diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 8f74b1f5..7f974dfc 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -8,7 +8,6 @@ import logging import socket -from urllib import urlencode from httplib import HTTPConnection, HTTPSConnection, HTTPException from Queue import Queue, Empty, Full from select import select @@ -23,14 +22,15 @@ except ImportError: BaseSSLError = None -from .filepost import encode_multipart_formdata +from .request import RequestMethods from .response import HTTPResponse from .exceptions import ( SSLError, MaxRetryError, TimeoutError, HostChangedError, - EmptyPoolError) + EmptyPoolError, +) log = logging.getLogger(__name__) @@ -38,7 +38,6 @@ log = logging.getLogger(__name__) _Default = object() - ## Connection objects (extension of httplib) class VerifiedHTTPSConnection(HTTPSConnection): @@ -46,11 +45,8 @@ class VerifiedHTTPSConnection(HTTPSConnection): Based on httplib.HTTPSConnection but wraps the socket with SSL certification. """ - - def __init__(self, **kwargs): - HTTPSConnection.__init__(self, **kwargs) - self.cert_reqs = None - self.ca_certs = None + cert_reqs = None + cert_reqs = None def set_cert(self, key_file=None, cert_file=None, cert_reqs='CERT_NONE', ca_certs=None): @@ -79,44 +75,48 @@ class VerifiedHTTPSConnection(HTTPSConnection): ## Pool objects class ConnectionPool(object): + """ + Base class for all connection pools, such as + :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`. + """ pass -class HTTPConnectionPool(ConnectionPool): +class HTTPConnectionPool(ConnectionPool, RequestMethods): """ Thread-safe connection pool for one host. - host + :param host: Host used for this HTTP Connection (e.g. "localhost"), passed into - httplib.HTTPConnection() + :class:`httplib.HTTPConnection`. - port + :param port: Port used for this HTTP Connection (None is equivalent to 80), passed - into httplib.HTTPConnection() + into :class:`httplib.HTTPConnection`. - strict + :param strict: Causes BadStatusLine to be raised if the status line can't be parsed as a valid HTTP/1.0 or 1.1 status line, passed into - httplib.HTTPConnection() + :class:`httplib.HTTPConnection`. - timeout + :param timeout: Socket timeout for each individual connection, can be a float. None disables timeout. - maxsize + :param maxsize: Number of connections to save that can be reused. More than 1 is useful in multithreaded situations. If ``block`` is set to false, more connections will be created but they will not be saved once they've been used. - block + :param block: If set to True, no more than ``maxsize`` connections will be used at a time. When no free connections are available, the call will block until a connection has been released. This is a useful side effect for particular multithreaded situations where one does not want to use more than maxsize connections per host to prevent flooding. - headers + :param headers: Headers to include with all requests, unless other headers are given explicitly. """ @@ -143,7 +143,7 @@ class HTTPConnectionPool(ConnectionPool): def _new_conn(self): """ - Return a fresh HTTPConnection. + Return a fresh :class:`httplib.HTTPConnection`. """ self.num_connections += 1 log.info("Starting new HTTP connection (%d): %s" % @@ -153,7 +153,14 @@ class HTTPConnectionPool(ConnectionPool): def _get_conn(self, timeout=None): """ Get a connection. Will return a pooled connection if one is available. - Otherwise, a fresh connection is returned. + + If no connections are available and :prop:`.block` is ``False``, then a + fresh connection is returned. + + :param timeout: + Seconds to wait before giving up and raising + :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and + :prop:`.block` is ``True``. """ conn = None try: @@ -176,6 +183,11 @@ class HTTPConnectionPool(ConnectionPool): def _put_conn(self, conn): """ Put a connection back into the pool. + + :param conn: + Connection object for the current host and port as returned by + :meth:`._new_conn` or :meth:`._get_conn`. + If the pool is already full, the connection is discarded because we exceeded maxsize. If connections are discarded frequently, then maxsize should be increased. @@ -198,12 +210,10 @@ class HTTPConnectionPool(ConnectionPool): if timeout is _Default: timeout = self.timeout - conn.request(method, url, **httplib_request_kw) conn.sock.settimeout(timeout) httplib_response = conn.getresponse() - log.debug("\"%s %s %s\" %s %s" % (method, url, conn._http_vsn_str, # pylint: disable-msg=W0212 @@ -213,6 +223,11 @@ class HTTPConnectionPool(ConnectionPool): def is_same_host(self, url): + """ + Check if the given ``url`` is a member of the same host as this + conncetion pool. + """ + # TODO: Add optional support for socket.gethostbyname checking. return (url.startswith('/') or get_host(url) == (self.scheme, self.host, self.port)) @@ -220,43 +235,50 @@ class HTTPConnectionPool(ConnectionPool): redirect=True, assert_same_host=True, timeout=_Default, pool_timeout=None, release_conn=None, **response_kw): """ - Get a connection from the pool and perform an HTTP request. + Get a connection from the pool and perform an HTTP request. This is the + lowest level call for making a request, so you'll need to specify all + the raw details. - method + .. note:: + + More commonly, it's appropriate to use a convenience method provided + by :class:`.RequestMethods`, such as :meth:`.request`. + + :param method: HTTP request method (such as GET, POST, PUT, etc.) - body + :param body: Data to send in the request body (useful for creating POST requests, see HTTPConnectionPool.post_url for more convenience). - headers + :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. - retries + :param retries: Number of retries to allow before raising a MaxRetryError exception. - redirect + :param redirect: Automatically handle redirects (status codes 301, 302, 303, 307), each redirect counts as a retry. - assert_same_host - If True, will make sure that the host of the pool requests is + :param assert_same_host: + If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When False, you can use the pool on an HTTP proxy and request foreign hosts. - timeout + :param timeout: If specified, overrides the default timeout for this one request. - pool_timeout + :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. - release_conn + :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received. This is useful if you're not preloading the response's content immediately. You will @@ -264,8 +286,9 @@ class HTTPConnectionPool(ConnectionPool): the connection back into the pool. If None, it takes the value of ``response_kw.get('preload_content', True)``. - Additional parameters are passed to - ``HTTPResponse.from_httplib(r, **response_kw)`` + :param \**response_kw: + Additional parameters are passed to + :meth:`urllib3.response.HTTPResponse.from_httplib` """ if headers is None: headers = self.headers @@ -291,11 +314,11 @@ class HTTPConnectionPool(ConnectionPool): # Request a connection from the queue # (Could raise SocketError: Bad file descriptor) conn = self._get_conn(timeout=pool_timeout) + # Make the request on the httplib connection object httplib_response = self._make_request(conn, method, url, timeout=timeout, body=body, headers=headers) - # print '!' # Import httplib's response into our own wrapper object response = HTTPResponse.from_httplib(httplib_response, @@ -348,66 +371,18 @@ class HTTPConnectionPool(ConnectionPool): return response - def get_url(self, url, fields=None, headers=None, retries=3, - redirect=True, **response_kw): - """ - Wrapper for performing GET with urlopen (see urlopen for more details). - - Supports an optional ``fields`` dictionary parameter key/value strings. - If provided, they will be added to the url. - """ - if fields: - url += '?' + urlencode(fields) - return self.urlopen('GET', url, headers=headers, retries=retries, - redirect=redirect, **response_kw) - - def post_url(self, url, fields=None, headers=None, retries=3, - redirect=True, encode_multipart=True, multipart_boundary=None, - **response_kw): - """ - Wrapper for performing POST with urlopen (see urlopen - for more details). - - Supports an optional ``fields`` parameter of key/value strings AND - key/filetuple. A filetuple is a (filename, data) tuple. For example: - - fields = { - 'foo': 'bar', - 'foofile': ('foofile.txt', 'contents of foofile'), - } - - If encode_multipart=True (default), then - ``urllib3.filepost.encode_multipart_formdata`` is used to encode the - payload with the appropriate content type. Otherwise - ``urllib.urlencode`` is used with 'application/x-www-form-urlencoded' - content type. - - Multipart encoding must be used when posting files, and it's reasonably - safe to use it other times too. It may break request signing, such as - OAuth. - - NOTE: If ``headers`` are supplied, the 'Content-Type' value will be - overwritten because it depends on the dynamic random boundary string - which is used to compose the body of the request. - """ - if encode_multipart: - body, content_type = encode_multipart_formdata(fields or {}, - boundary=multipart_boundary) - else: - body, content_type = ( - urlencode(fields or {}), - 'application/x-www-form-urlencoded') - - headers = headers or {} - headers.update({'Content-Type': content_type}) - - return self.urlopen('POST', url, body, headers=headers, - retries=retries, redirect=redirect, **response_kw) - class HTTPSConnectionPool(HTTPConnectionPool): """ - Same as HTTPConnectionPool, but HTTPS. + Same as :class:`.HTTPConnectionPool`, but HTTPS. + + When Python is compiled with the :mod:`ssl` module, then + :class:`.VerifiedHTTPSConnection` is used, which *can* verify certificates, + instead of :class:httplib.HTTPSConnection`. + + The ``key_file``, ``cert_file``, ``cert_reqs``, and ``ca_certs`` parameters + are only used if :mod:`ssl` is available and are fed into + :meth:`ssl.wrap_socket` to upgrade the connection socket into an SSL socket. """ scheme = 'https' @@ -416,7 +391,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): strict=False, timeout=None, maxsize=1, block=False, headers=None, key_file=None, cert_file=None, - cert_reqs=ssl.CERT_REQUIRED, ca_certs=None): + cert_reqs='CERT_NONE', ca_certs=None): super(HTTPSConnectionPool, self).__init__(host, port, strict, timeout, maxsize, @@ -428,9 +403,8 @@ class HTTPSConnectionPool(HTTPConnectionPool): def _new_conn(self): """ - Return a fresh HTTPSConnection. + Return a fresh :class:`httplib.HTTPSConnection`. """ - self.num_connections += 1 log.info("Starting new HTTPS connection (%d): %s" % (self.num_connections, self.host)) @@ -451,22 +425,29 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, """ Shortcuts for generating request headers. - keep_alive - If true, adds 'connection: keep-alive' header. + :param keep_alive: + If ``True``, adds 'connection: keep-alive' header. - accept_encoding + :param accept_encoding: Can be a boolean, list, or string. - True translates to 'gzip,deflate'. + ``True`` translates to 'gzip,deflate'. List will get joined by comma. String will be used as provided. - user_agent + :param user_agent: String representing the user-agent you want, such as "python-urllib3/0.6" - basic_auth + :param basic_auth: Colon-separated username:password string for 'authorization: basic ...' auth header. + + Example: :: + + >>> make_headers(keep_alive=True, user_agent="Batman/1.0") + {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} + >>> make_headers(accept_encoding=True) + {'accept-encoding': 'gzip,deflate'} """ headers = {} if accept_encoding: @@ -495,11 +476,12 @@ def get_host(url): """ Given a url, return its scheme, host and port (None if it's not there). - For example: - >>> get_host('http://google.com/mail/') - http, google.com, None - >>> get_host('google.com:80') - http, google.com, 80 + For example: :: + + >>> get_host('http://google.com/mail/') + ('http', 'google.com', None) + >>> get_host('google.com:80') + ('http', 'google.com', 80) """ # This code is actually similar to urlparse.urlsplit, but much # simplified for our needs. @@ -517,13 +499,23 @@ def get_host(url): def connection_from_url(url, **kw): """ - Given a url, return an HTTP(S)ConnectionPool instance of its host. + Given a url, return an :class:`.ConnectionPool` instance of its host. - This is a shortcut for not having to determine the host of the url - before creating an HTTP(S)ConnectionPool instance. + This is a shortcut for not having to parse out the scheme, host, and port + of the url before creating an :class:`.ConnectionPool` instance. - Passes on whatever kw arguments to the constructor of - HTTP(S)ConnectionPool. (e.g. timeout, maxsize, block) + :param url: + Absolute URL string that must include the scheme. Port is optional. + + :param \**kw: + Passes additional parameters to the constructor of the appropriate + :class:`.ConnectionPool`. Useful for specifying things like + timeout, maxsize, headers, etc. + + Example: :: + + >>> conn = connection_from_url('http://google.com/') + >>> r = conn.request('GET', '/') """ scheme, host, port = get_host(url) if scheme == 'https': diff --git a/requests/packages/urllib3/filepost.py b/requests/packages/urllib3/filepost.py index 8e65b5c2..2ffea8bb 100644 --- a/requests/packages/urllib3/filepost.py +++ b/requests/packages/urllib3/filepost.py @@ -22,6 +22,19 @@ def get_content_type(filename): def encode_multipart_formdata(fields, boundary=None): + """ + Encode a dictionary of ``fields`` using the multipart/form-data mime format. + + :param fields: + Dictionary of fields. The key is treated as the field name, and the + value as the body of the form-data. If the value is a tuple of two + elements, then the first element is treated as the filename of the + form-data section. + + :param boundary: + If not specified, then a random boundary will be generated using + :func:`mimetools.choose_boundary`. + """ body = StringIO() if boundary is None: boundary = mimetools.choose_boundary() diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index 1d757d19..622789e5 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -5,7 +5,17 @@ # the MIT License: http://www.opensource.org/licenses/mit-license.php from ._collections import RecentlyUsedContainer -from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, get_host +from .connectionpool import ( + HTTPConnectionPool, HTTPSConnectionPool, + get_host, connection_from_url, +) + + +__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] + + +from .request import RequestMethods +from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool pool_classes_by_scheme = { @@ -19,16 +29,27 @@ port_by_scheme = { } -class PoolManager(object): +class PoolManager(RequestMethods): """ Allows for arbitrary requests while transparently keeping track of necessary connection pools for you. - num_pools + :param num_pools: Number of connection pools to cache before discarding the least recently used pool. - Additional parameters are used to create fresh ConnectionPool instances. + :param \**connection_pool_kw: + Additional parameters are used to create fresh + :class:`urllib3.connectionpool.ConnectionPool` instances. + + Example: :: + + >>> manager = PoolManager() + >>> r = manager.urlopen("http://google.com/") + >>> r = manager.urlopen("http://google.com/mail") + >>> r = manager.urlopen("http://yahoo.com/") + >>> len(r.pools) + 2 """ @@ -36,17 +57,17 @@ class PoolManager(object): def __init__(self, num_pools=10, **connection_pool_kw): self.connection_pool_kw = connection_pool_kw - self.pools = RecentlyUsedContainer(num_pools) - self.recently_used_pools = [] def connection_from_host(self, host, port=80, scheme='http'): """ - Get a ConnectionPool based on the host, port, and scheme. + Get a :class:`ConnectionPool` based on the host, port, and scheme. + + Note that an appropriate ``port`` value is required here to normalize + connection pools in our container most effectively. """ pool_key = (scheme, host, port) - # If the scheme, host, or port doesn't match existing open connections, # open a new ConnectionPool. pool = self.pools.get(pool_key) @@ -63,19 +84,45 @@ class PoolManager(object): def connection_from_url(self, url): """ - Similar to connectionpool.connection_from_url but doesn't pass any - additional keywords to the ConnectionPool constructor. Additional - keywords are taken from the PoolManager constructor. + Similar to :func:`urllib3.connectionpool.connection_from_url` but + doesn't pass any additional parameters to the + :class:`urllib3.connectionpool.ConnectionPool` constructor. + + Additional parameters are taken from the :class:`.PoolManager` + constructor. """ scheme, host, port = get_host(url) port = port or port_by_scheme.get(scheme, 80) - r = self.connection_from_host(host, port=port, scheme=scheme) + return self.connection_from_host(host, port=port, scheme=scheme) - return r + def urlopen(self, method, url, **kw): + """ + Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`. + + ``url`` must be absolute, such that an appropriate + :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it. + """ + conn = self.connection_from_url(url) + return conn.urlopen(method, url, assert_same_host=False, **kw) + + +class ProxyManager(object): + """ + Given a ConnectionPool to a proxy, the ProxyManager's ``urlopen`` method + will make requests to any url through the defined proxy. + """ + + def __init__(self, proxy_pool): + self.proxy_pool = proxy_pool def urlopen(self, method, url, **kw): "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." - conn = self.connection_from_url(url) - return conn.urlopen(method, url, **kw) + kw['assert_same_host'] = False + return self.proxy_pool.urlopen(method, url, **kw) + + +def proxy_from_url(url, **pool_kw): + proxy_pool = connection_from_url(url, **pool_kw) + return ProxyManager(proxy_pool) diff --git a/requests/packages/urllib3/request.py b/requests/packages/urllib3/request.py new file mode 100644 index 00000000..a7e0b5de --- /dev/null +++ b/requests/packages/urllib3/request.py @@ -0,0 +1,145 @@ +# urllib3/request.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + + +from urllib import urlencode + +from .filepost import encode_multipart_formdata + + +__all__ = ['RequestMethods'] + + +class RequestMethods(object): + """ + Convenience mixin for classes who implement a :meth:`urlopen` method, such + as :class:`~urllib3.connectionpool.HTTPConnectionPool` and + :class:`~urllib3.poolmanager.PoolManager`. + + Provides behavior for making common types of HTTP request methods and + decides which type of request field encoding to use. + + Specifically, + + :meth:`.request_encode_url` is for sending requests whose fields are encoded + in the URL (such as GET, HEAD, DELETE). + + :meth:`.request_encode_body` is for sending requests whose fields are + encoded in the *body* of the request using multipart or www-orm-urlencoded + (such as for POST, PUT, PATCH). + + :meth:`.request` is for making any kind of request, it will look up the + appropriate encoding format and use one of the above two methods to make + the request. + """ + + _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS']) + + _encode_body_methods = set(['PATCH', 'POST', 'PUT', 'TRACE']) + + def urlopen(self, method, url, body=None, headers=None, + encode_multipart=True, multipart_boundary=None, + **kw): + raise NotImplemented("Classes extending RequestMethods must implement " + "their own ``urlopen`` method.") + + def request(self, method, url, fields=None, headers=None, **urlopen_kw): + """ + Make a request using :meth:`urlopen` with the appropriate encoding of + ``fields`` based on the ``method`` used. + + This is a convenience method that requires the least amount of manual + effort. It can be used in most situations, while still having the option + to drop down to more specific methods when necessary, such as + :meth:`request_encode_url`, :meth:`request_encode_body`, + or even the lowest level :meth:`urlopen`. + """ + method = method.upper() + + if method in self._encode_url_methods: + return self.request_encode_url(method, url, fields=fields, + headers=headers, + **urlopen_kw) + else: + return self.request_encode_body(method, url, fields=fields, + headers=headers, + **urlopen_kw) + + def request_encode_url(self, method, url, fields=None, **urlopen_kw): + """ + Make a request using :meth:`urlopen` with the ``fields`` encoded in + the url. This is useful for request methods like GET, HEAD, DELETE, etc. + """ + if fields: + url += '?' + urlencode(fields) + return self.urlopen(method, url, **urlopen_kw) + + def request_encode_body(self, method, url, fields=None, headers=None, + encode_multipart=True, multipart_boundary=None, + **urlopen_kw): + """ + Make a request using :meth:`urlopen` with the ``fields`` encoded in + the body. This is useful for request methods like POST, PUT, PATCH, etc. + + When ``encode_multipart=True`` (default), then + :meth:`urllib3.filepost.encode_multipart_formdata` is used to encode the + payload with the appropriate content type. Otherwise + :meth:`urllib.urlencode` is used with the + 'application/x-www-form-urlencoded' content type. + + Multipart encoding must be used when posting files, and it's reasonably + safe to use it in other times too. However, it may break request signing, + such as with OAuth. + + Supports an optional ``fields`` parameter of key/value strings AND + key/filetuple. A filetuple is a (filename, data) tuple. For example: :: + + fields = { + 'foo': 'bar', + 'fakefile': ('foofile.txt', 'contents of foofile'), + 'realfile': ('barfile.txt', open('realfile').read()), + 'nonamefile': ('contents of nonamefile field'), + } + + When uploading a file, providing a filename (the first parameter of the + tuple) is optional but recommended to best mimick behavior of browsers. + + Note that if ``headers`` are supplied, the 'Content-Type' header will be + overwritten because it depends on the dynamic random boundary string + which is used to compose the body of the request. The random boundary + string can be explicitly set with the ``multipart_boundary`` parameter. + """ + if encode_multipart: + body, content_type = encode_multipart_formdata(fields or {}, + boundary=multipart_boundary) + else: + body, content_type = (urlencode(fields or {}), + 'application/x-www-form-urlencoded') + + headers = headers or {} + headers.update({'Content-Type': content_type}) + + return self.urlopen(method, url, body=body, headers=headers, + **urlopen_kw) + + # Deprecated: + + def get_url(self, url, fields=None, **urlopen_kw): + """ + .. deprecated:: 1.0 + Use :meth:`request` instead. + """ + return self.request_encode_url('GET', url, fields=fields, + **urlopen_kw) + + def post_url(self, url, fields=None, headers=None, **urlopen_kw): + """ + .. deprecated:: 1.0 + Use :meth:`request` instead. + """ + return self.request_encode_body('POST', url, fields=fields, + headers=headers, + **urlopen_kw) diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 2bbc06ab..4cd15c11 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -42,15 +42,15 @@ class HTTPResponse(object): Extra parameters for behaviour not present in httplib.HTTPResponse: - preload_content + :param preload_content: If True, the response's body will be preloaded during construction. - decode_content + :param decode_content: If True, attempts to decode specific content-encoding's based on headers (like 'gzip' and 'deflate') will be skipped and raw data will be used instead. - original_response + :param original_response: When this HTTPResponse wrapper is generated from an httplib.HTTPResponse object, it's convenient to include the original for debug purposes. It's otherwise unused. @@ -103,18 +103,19 @@ class HTTPResponse(object): def read(self, amt=None, decode_content=True, cache_content=False): """ - Similar to ``httplib.HTTPResponse.read(amt=None)``. + Similar to :meth:`httplib.HTTPResponse.read`, but with two additional + parameters: ``decode_content`` and ``cache_content``. - amt + :param amt: How much of the content to read. If specified, decoding and caching is skipped because we can't decode partial content nor does it make sense to cache partial content as the full response. - decode_content + :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. (Overridden if ``amt`` is set.) - cache_content + :param cache_content: If True, will save the returned data such that the same result is returned despite of the state of the underlying file object. This is useful if you want the ``.data`` property to continue working @@ -156,8 +157,8 @@ class HTTPResponse(object): @staticmethod def from_httplib(r, **response_kw): """ - Given an httplib.HTTPResponse instance ``r``, return a corresponding - urllib3.HTTPResponse object. + Given an :class:`httplib.HTTPResponse` instance ``r``, return a + corresponding :class:`urllib3.response.HTTPResponse` object. Remaining parameters are passed to the HTTPResponse constructor, along with ``original_response=r``. From e7b31af784c25b1cbc390bad60ea19f24a0c7905 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 10 Oct 2011 20:59:10 -0400 Subject: [PATCH 148/255] urllib3 fix --- requests/packages/urllib3/connectionpool.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 7f974dfc..8da425a0 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -258,8 +258,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): these headers completely replace any pool-specific headers. :param retries: - Number of retries to allow before raising - a MaxRetryError exception. + Number of retries to allow before raising a MaxRetryError exception. :param redirect: Automatically handle redirects (status codes 301, 302, 303, 307), @@ -320,16 +319,19 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): timeout=timeout, body=body, headers=headers) + # If we're going to release the connection in ``finally:``, then + # the request doesn't need to know about the connection. Otherwise + # it will also try to release it and we'll have a double-release + # mess. + response_conn = not release_conn and conn + + # Import httplib's response into our own wrapper object response = HTTPResponse.from_httplib(httplib_response, pool=self, - connection=conn, + connection=response_conn, **response_kw) - if release_conn: - # The connection will be released manually in the ``finally:`` - response.connection = None - # else: # The connection will be put back into the pool when # ``response.release_conn()`` is called (implicitly by From c8f2b3a722d95be95036da13dd9a2a363ed2cd10 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 10 Oct 2011 21:05:54 -0400 Subject: [PATCH 149/255] undo ref trickery --- requests/models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index 78290570..27cae2a5 100644 --- a/requests/models.py +++ b/requests/models.py @@ -241,8 +241,7 @@ class Request(object): self.response = r # Give Response some context. - self.response.request = ref(self)() - self.response.request.response = ref(self.response)() + self.response.request = self def send(self, anyway=False): @@ -310,7 +309,7 @@ class Request(object): pools = self._pools # Part of a connection pool, so no fancy stuff. Sorry! - do_block = True + do_block = False if self.cookies: # Skip if 'cookie' header is explicitly set. From 4dc48aea6320ab23de1b2f3adecfc68c4013e407 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 10 Oct 2011 21:08:17 -0400 Subject: [PATCH 150/255] remove logger for urllib3 --- requests/core.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/requests/core.py b/requests/core.py index 613e3324..9e199211 100644 --- a/requests/core.py +++ b/requests/core.py @@ -24,8 +24,6 @@ __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' -import logging -logging.basicConfig() from api import * from exceptions import * From fedbafc79b73175b5ec4b75dccf189da426ef134 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 10 Oct 2011 21:08:30 -0400 Subject: [PATCH 151/255] v0.7.0 dev --- requests/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/core.py b/requests/core.py index 9e199211..a3fdb728 100644 --- a/requests/core.py +++ b/requests/core.py @@ -18,7 +18,7 @@ This module implements the main Requests system. __title__ = 'requests' -__version__ = '0.6.2 (dev)' +__version__ = '0.7.0 (dev)' __build__ = 0x000602 __author__ = 'Kenneth Reitz' __license__ = 'ISC' From f524fe4934d41fc08173cd61f49c385207d42390 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 11 Oct 2011 08:05:11 -0400 Subject: [PATCH 152/255] python version support update --- docs/community/faq.rst | 18 +++++++++++++++++- docs/user/intro.rst | 19 ------------------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/docs/community/faq.rst b/docs/community/faq.rst index 5671d3b1..91ca06b2 100644 --- a/docs/community/faq.rst +++ b/docs/community/faq.rst @@ -51,7 +51,23 @@ Chris Adams gave an excellent summary on Python 3 Support? ----------------- -It's on the way. Here's a list of `supported interpreters `_. +It's on the way. Here's a list of Python platforms that are officially +supported: + +* cPython 2.5 +* cPython 2.5.5 +* cPython 2.5.6 +* cPython 2.6 +* cPython 2.6.6 +* cPython 2.6.7 +* cPython 2.7 +* cPython 2.7.1 +* cPython 2.7.2 +* PyPy-c 1.4 +* PyPy-c 1.5 + + +Support for Python 3.x is coming *very* soon. Keep-alive Support? diff --git a/docs/user/intro.rst b/docs/user/intro.rst index 9837fa14..c2b6bf36 100644 --- a/docs/user/intro.rst +++ b/docs/user/intro.rst @@ -48,23 +48,4 @@ Requests License THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -.. _interpreters: -Python Interpreters -------------------- -At this time, the following Python platforms are officially supported: - -* cPython 2.5 -* cPython 2.5.5 -* cPython 2.5.6 -* cPython 2.6 -* cPython 2.6.6 -* cPython 2.6.7 -* cPython 2.7 -* cPython 2.7.1 -* cPython 2.7.2 -* PyPy-c 1.4 -* PyPy-c 1.5 - - -Support for Python 3.x is planned. From 3cfc83b23bb274cb8ad0160acd548b2401fee5f2 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 11 Oct 2011 08:39:43 -0400 Subject: [PATCH 153/255] merge cleanups --- requests/api.py | 7 ++----- requests/models.py | 3 ++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/requests/api.py b/requests/api.py index 6b1839a7..696305e5 100644 --- a/requests/api.py +++ b/requests/api.py @@ -46,10 +46,7 @@ def request(method, url, method = str(method).upper() config = get_config(config) - cookies = cookiejar_from_dict(cookies if cookies is not None else dict()) - - cookies = cookiejar_from_dict(cookies) - + # cookies = cookiejar_from_dict(cookies if cookies is not None else dict()) # cookies = cookiejar_from_dict(cookies) # Expand header values @@ -72,7 +69,7 @@ def request(method, url, proxies=proxies or config.get('proxies'), _pools=_pools ) - + hooks = setup_hooks(hooks if hooks is not None else dict()) # Arguments manipulation hook. diff --git a/requests/models.py b/requests/models.py index 2c863b88..e27e1bd3 100644 --- a/requests/models.py +++ b/requests/models.py @@ -90,7 +90,7 @@ class Request(object): #: :class:`AuthObject` to attach to :class:`Request `. self.auth = auth - #: CookieJar to attach to :class:`Request `. + #: CookieDict to attach to :class:`Request `. self.cookies = cookies #: True if Request has been sent. @@ -316,6 +316,7 @@ class Request(object): if 'cookie' not in self.headers: # Simple cookie with our dict. + # TODO: Multi-value headers. c = SimpleCookie() c.load(self.cookies) From abe33f0ef4b9ea9588eca01f902a534f0acd0cde Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 11 Oct 2011 08:40:32 -0400 Subject: [PATCH 154/255] AUTHORS += 'Luca De Vitis' --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 1a4a5fb7..4a1f5c62 100644 --- a/AUTHORS +++ b/AUTHORS @@ -54,3 +54,4 @@ Patches and Suggestions - Serge Domkowski - Daniel Miller - Pat Nakajima +- Luca De Vitis From 20c930e48e31916481d6d45bcbb079f7c4670b18 Mon Sep 17 00:00:00 2001 From: Douglas Morrison Date: Wed, 12 Oct 2011 12:57:34 +0000 Subject: [PATCH 155/255] Fix errors introduced by merging an outdated pull request. Don't do that anymore, OK? --- requests/api.py | 16 ++--- requests/config.py | 80 ----------------------- requests/hooks.py | 157 ++------------------------------------------- requests/models.py | 26 +------- 4 files changed, 18 insertions(+), 261 deletions(-) delete mode 100644 requests/config.py diff --git a/requests/api.py b/requests/api.py index 696305e5..0417f81d 100644 --- a/requests/api.py +++ b/requests/api.py @@ -14,7 +14,7 @@ This module implements the Requests API. from ._config import get_config from .models import Request, Response from .status_codes import codes -from hooks import setup_hooks, dispatch_hooks +from .hooks import dispatch_hooks from .utils import cookiejar_from_dict, header_expand @@ -46,7 +46,9 @@ def request(method, url, method = str(method).upper() config = get_config(config) - # cookies = cookiejar_from_dict(cookies if cookies is not None else dict()) + if cookies is None: + cookies = {} + # cookies = cookiejar_from_dict(cookies) # Expand header values @@ -70,16 +72,14 @@ def request(method, url, _pools=_pools ) - hooks = setup_hooks(hooks if hooks is not None else dict()) - # Arguments manipulation hook. - args = dispatch_hooks(hooks['args'], args) + args = dispatch_hooks('args', hooks, args) # Create Request object. r = Request(**args) # Pre-request hook. - r = dispatch_hooks(hooks['pre_request'], r) + r = dispatch_hooks('pre_request', hooks, r) # Only construct the request (for async) if _return_request: @@ -90,10 +90,10 @@ def request(method, url, # TODO: Add these hooks inline. # Post-request hook. - r = dispatch_hooks(hooks['post_request'], r) + r = dispatch_hooks('post_request', hooks, r) # Response manipulation hook. - r.response = dispatch_hooks(hooks['response'], r.response) + r.response = dispatch_hooks('response', hooks, r.response) return r.response diff --git a/requests/config.py b/requests/config.py deleted file mode 100644 index e6adfca6..00000000 --- a/requests/config.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -requests.config -~~~~~~~~~~~~~~~ - -This module provides the Requests settings feature set. - -""" - -class Settings(object): - _singleton = {} - - # attributes with defaults - __attrs__ = [] - - def __init__(self, **kwargs): - super(Settings, self).__init__() - - self.__dict__ = self._singleton - - - def __call__(self, *args, **kwargs): - # new instance of class to call - r = self.__class__() - - # cache previous settings for __exit__ - r.__cache = self.__dict__.copy() - map(self.__cache.setdefault, self.__attrs__) - - # set new settings - self.__dict__.update(*args, **kwargs) - - return r - - - def __enter__(self): - pass - - - def __exit__(self, *args): - - # restore cached copy - self.__dict__.update(self.__cache.copy()) - del self.__cache - - - def __getattribute__(self, key): - if key in object.__getattribute__(self, '__attrs__'): - try: - return object.__getattribute__(self, key) - except AttributeError: - return None - return object.__getattribute__(self, key) - - -settings = Settings() - -settings.base_headers = { - 'User-Agent': 'python-requests.org', - 'Accept-Encoding': ', '.join([ 'identity', 'deflate', 'compress', 'gzip' ]), -} -settings.accept_gzip = True -settings.proxies = None -settings.verbose = None -settings.timeout = None -settings.max_redirects = 30 -settings.decode_unicode = False -settings.gracefull_hooks = True - -#: A dictionary of default hooks to be applied, based on settings. -settings.default_hooks = { - 'args': list(), - 'pre_request': list(), - 'post_request': list(), - 'response': list() -} - -#: Use socket.setdefaulttimeout() as fallback? -settings.timeout_fallback = True diff --git a/requests/hooks.py b/requests/hooks.py index e2980ac3..cd113b97 100644 --- a/requests/hooks.py +++ b/requests/hooks.py @@ -23,162 +23,19 @@ Available hooks: """ import warnings -from collections import Iterable -import config -import zlib -import bz2 -from cgi import parse_header +import collections -def setup_hooks(supplied): - """Setup a hooks mapping, based on the supplied argument. Eache mapping - value will be list of hooks that will extend the **default_hooks**. - :param supplied: a dictionary of hooks. Each value can either be a callable - or a list of callables. - :type supplied: dict - :returns: a dictionary of hooks that extends the **default_hooks** dictionary. - :rtype: dict - """ +def dispatch_hooks(key, hooks, hook_data): + """Dipatches a hook dictionary on a given peice of data.""" - # Copy the default hooks settings. - default = config.settings.default_hooks - dispatching = dict([(k, v[:]) for k, v in default.items()]) + hooks = (hooks or {}).get(key, []) + hooks = hooks if isinstance(hooks, collections.Iterable) else [hooks] - # I abandoned the idea of a dictionary of sets because sets may not keep - # insertion order, while it may be important. Also, there is no real reason - # to force hooks to run once. - for hooks, values in supplied.items(): - hook_list = values if isinstance(values, Iterable) else [values] - dispatching[hooks].extend(hook_list) - - # If header is set by config, maybe response is encoded. - if config.settings.base_headers.get('Accept-Encoding', ''): - if not decode_encoding in dispatching['response']: - # It's safer to put decoding as first hook. - dispatching['response'].insert(0, decode_encoding) - - if config.settings.decode_unicode: - try: - # Try unicode encoding just after content decoding... - index = dispatching['response'].index(decode_encoding) + 1 - except ValueError: - # ... Or as first hook - index = 0 - dispatching['response'].insert(index, decode_unicode) - - return dispatching - -def dispatch_hooks(hooks, data): - """Dispatches multiple hooks on a given piece of data. - - :param key: the hooks group to lookup - :type key: str - :param hooks: the hooks dictionary. The value of each key can be a callable - object, or a list of callable objects. - :type hooks: dict - :param data: the object on witch the hooks should be applied - :type data: object - """ for hook in hooks: try: - # hook must be a callable. - data = hook(data) - + hook_data = hook(hook_data) or hook_data except Exception, why: - - # Letting users to choose a policy may be an idea. It can be as - # simple as "be gracefull, or not": - # - # config.settings.gracefull_hooks = True | False - if not config.settings.gracefull_hooks: raise - warnings.warn(str(why)) - return data - -#: Example response hook that turns a JSON formatted -#: :py:class:`requests.models.Response.content` into a dumped data structure:: -#: -#: try: -#: import json -#: except ImportError: -#: try: -#: import simplejson as json -#: except ImportError: -#: json = False -#: -#: if json: -#: def json_content(r): -#: """Turns content into a dumped JSON structure.""" -#: r._content = json.dumps(r.content) -#: return r -#: -#: Example response hook that turns an XML formatted -#: :py:class:`requests.models.Response.content` into an ElementTree:: -#: -#: try: -#: from lxml import etree -#: except ImportError: -#: try: -#: import xml.etree.cElementTree as etree -#: except ImportError: -#: try: -#: import xml.etree.ElementTree as etree -#: except ImportError: -#: try: -#: import cElementTree as etree -#: except ImportError: -#: try: -#: import elementtree.ElementTree as etree -#: except ImportError: -#: etree = False -#: -#: if etree: -#: def etree_content(r): -#: """Turns content into an ElementTree structure.""" -#: r._content = etree.fromstring(r.content) -#: return r - -def decode_unicode(r): - """Encode content into unicode string. - - :param r: response object - :type r: :py:class:`requests.models.Response` - :returns: the same input object. - :rtype: :py:class:`requests.models.Response` - """ - content_type, params = parse_header(r.headers['content-type']) - charset = params.get('charset', '').strip("'\"") - r._content = unicode(r.content, charset) if charset else unicode(r.content) - return r - -def decode_encoding(r): - """Decode content using Contetn-Encoding header. - - :param r: response object - :type r: :py:class:`requests.models.Response` - :returns: the same input object. - :rtype: :py:class:`requests.models.Response` - """ - - # Dictionary of content decoders. - decode = { - # No decoding applied. - 'identity': lambda content: content, - # Decode Response content compressed with deflate. - 'deflate': lambda content: zlib.decompress(content), - # Decode Response content compressed with gzip. - 'gzip': lambda content: zlib.decompress(content, 16+zlib.MAX_WBITS), - # Decode Response content compressed with bz2. - # Not a standard Content-Encoding value, but.. - 'bzip2': lambda content: bz2.decompress(content), - } - # Decode Response content compressed with compress. - # If I understood zlib... - decode['compress'] = decode['deflate'] - - # Apply decoding only if the header is set. - encoding = r.headers['content-encoding'] - if encoding: - r._content = decode[encoding](r.content) - return r + return hook_data diff --git a/requests/models.py b/requests/models.py index e27e1bd3..5383a70e 100644 --- a/requests/models.py +++ b/requests/models.py @@ -453,21 +453,14 @@ class Response(object): (if available). """ - if self._content is None: - # Read the contents. - self._content = self.fo.read() + if self._content is not None: + return self._content if self._content_consumed: raise RuntimeError( 'The content for this response was already consumed') - # # Decode GZip'd content. - # if 'gzip' in self.headers.get('content-encoding', ''): - # try: - # self._content = decode_gzip(self._content) - # except zlib.error: - # pass - self._content = self.fo.read() + # Read the contents. self._content = self.raw.read() or self.raw.data # Decode GZip'd content. @@ -477,22 +470,9 @@ class Response(object): except zlib.error: pass - # Decode unicode content. - if settings.decode_unicode: - self._content = get_unicode_from_response(self) - # Decode GZip'd content. - if 'gzip' in self.headers.get('content-encoding', ''): - try: - self._content = decode_gzip(self._content) - except zlib.error: - pass - # Decode unicode content. if self.config.get('decode_unicode'): self._content = get_unicode_from_response(self) - # # Decode unicode content. - # if settings.decode_unicode: - # self._content = get_unicode_from_response(self) self._content_consumed = True return self._content From c4f013a6f999c2e945e00b1f052074b9c9377842 Mon Sep 17 00:00:00 2001 From: Douglas Morrison Date: Wed, 12 Oct 2011 13:21:09 +0000 Subject: [PATCH 156/255] Achieve full API compatibility. --- requests/api.py | 10 +++++----- requests/hooks.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requests/api.py b/requests/api.py index 0417f81d..dea4c1ad 100644 --- a/requests/api.py +++ b/requests/api.py @@ -14,7 +14,7 @@ This module implements the Requests API. from ._config import get_config from .models import Request, Response from .status_codes import codes -from .hooks import dispatch_hooks +from .hooks import dispatch_hook from .utils import cookiejar_from_dict, header_expand @@ -73,13 +73,13 @@ def request(method, url, ) # Arguments manipulation hook. - args = dispatch_hooks('args', hooks, args) + args = dispatch_hook('args', hooks, args) # Create Request object. r = Request(**args) # Pre-request hook. - r = dispatch_hooks('pre_request', hooks, r) + r = dispatch_hook('pre_request', hooks, r) # Only construct the request (for async) if _return_request: @@ -90,10 +90,10 @@ def request(method, url, # TODO: Add these hooks inline. # Post-request hook. - r = dispatch_hooks('post_request', hooks, r) + r = dispatch_hook('post_request', hooks, r) # Response manipulation hook. - r.response = dispatch_hooks('response', hooks, r.response) + r.response = dispatch_hook('response', hooks, r.response) return r.response diff --git a/requests/hooks.py b/requests/hooks.py index cd113b97..36ec3c06 100644 --- a/requests/hooks.py +++ b/requests/hooks.py @@ -26,7 +26,7 @@ import warnings import collections -def dispatch_hooks(key, hooks, hook_data): +def dispatch_hook(key, hooks, hook_data): """Dipatches a hook dictionary on a given peice of data.""" hooks = (hooks or {}).get(key, []) From a0e9b21c9da518d757bb5bd7510b50e6095de85c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 13 Oct 2011 14:34:56 -0400 Subject: [PATCH 157/255] blargh --- requests/_config.py | 14 ++++++++++++-- requests/utils.py | 2 ++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/requests/_config.py b/requests/_config.py index 7ecc2a10..d30b70b8 100644 --- a/requests/_config.py +++ b/requests/_config.py @@ -39,8 +39,12 @@ def get_config(config=None, default_config=None): # Module-level defaults. defaults = dict() -defaults['base_headers'] = {'User-Agent': 'python-requests.org'} -defaults['accept_gzip'] = True +defaults['base_headers'] = { + 'User-Agent': 'python-requests.org', + 'Accept-Encoding': ', '.join([ 'identity', 'deflate', 'compress', 'gzip' ]), +} + +# defaults['accept_gzip'] = True defaults['proxies'] = {} defaults['verbose'] = None defaults['timeout'] = None @@ -50,3 +54,9 @@ defaults['keep_alive'] = True defaults['max_connections'] = 10 +defaults['hooks'] = { + 'args': list(), + 'pre_request': list(), + 'post_request': list(), + 'response': list() +} \ No newline at end of file diff --git a/requests/utils.py b/requests/utils.py index 832a11f3..43b6df70 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -280,6 +280,8 @@ def get_unicode_from_response(r): 3. fall back and replace all unicode characters """ + # TODO: DEBUG + return r.content tried_encodings = [] From b908559ac8cf016d4217b20a020d540a377f1b12 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 13 Oct 2011 19:41:40 -0400 Subject: [PATCH 158/255] urllib2 => httplib --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 3f6455ed..30fa8f15 100644 --- a/README.rst +++ b/README.rst @@ -27,7 +27,7 @@ See `the same code, without Requests `_. Requests allow you to send **HEAD**, **GET**, **POST**, **PUT**, **PATCH**, and **DELETE** HTTP requests. You can add headers, form data, multipart files, and parameters with simple Python dictionaries, and access the -response data in the same way. It's powered by urllib2, but it does +response data in the same way. It's powered by httplib, but it does all the hard work and crazy hacks for you. From 7c271a466fe557f7c883f6105728f73f29cdb293 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 16:18:52 -0400 Subject: [PATCH 159/255] Import urllib3 --- requests/packages/urllib3/__init__.py | 48 ++ requests/packages/urllib3/_collections.py | 120 ++++ requests/packages/urllib3/connectionpool.py | 525 ++++++++++++++++++ requests/packages/urllib3/contrib/__init__.py | 0 requests/packages/urllib3/contrib/ntlmpool.py | 117 ++++ requests/packages/urllib3/exceptions.py | 35 ++ requests/packages/urllib3/filepost.py | 71 +++ requests/packages/urllib3/poolmanager.py | 128 +++++ requests/packages/urllib3/request.py | 145 +++++ requests/packages/urllib3/response.py | 181 ++++++ 10 files changed, 1370 insertions(+) create mode 100644 requests/packages/urllib3/__init__.py create mode 100644 requests/packages/urllib3/_collections.py create mode 100644 requests/packages/urllib3/connectionpool.py create mode 100644 requests/packages/urllib3/contrib/__init__.py create mode 100644 requests/packages/urllib3/contrib/ntlmpool.py create mode 100644 requests/packages/urllib3/exceptions.py create mode 100644 requests/packages/urllib3/filepost.py create mode 100644 requests/packages/urllib3/poolmanager.py create mode 100644 requests/packages/urllib3/request.py create mode 100644 requests/packages/urllib3/response.py diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py new file mode 100644 index 00000000..7a2d0186 --- /dev/null +++ b/requests/packages/urllib3/__init__.py @@ -0,0 +1,48 @@ +# urllib3/__init__.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +""" +urllib3 - Thread-safe connection pooling and re-using. +""" + +__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' +__license__ = 'MIT' +__version__ = '1.0.1' + + +from .connectionpool import ( + HTTPConnectionPool, + HTTPSConnectionPool, + connection_from_url, + get_host, + make_headers) + + +from .exceptions import ( + HTTPError, + MaxRetryError, + SSLError, + TimeoutError) + +from .poolmanager import PoolManager, ProxyManager, proxy_from_url +from .response import HTTPResponse +from .filepost import encode_multipart_formdata + + +# Set default logging handler to avoid "No handler found" warnings. +import logging +try: + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + +logging.getLogger(__name__).addHandler(NullHandler()) + +# ... Clean up. +del logging +del NullHandler diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py new file mode 100644 index 00000000..bd01da33 --- /dev/null +++ b/requests/packages/urllib3/_collections.py @@ -0,0 +1,120 @@ +# urllib3/_collections.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +from collections import MutableMapping, deque + + +__all__ = ['RecentlyUsedContainer'] + + +class AccessEntry(object): + __slots__ = ('key', 'is_valid') + + def __init__(self, key, is_valid=True): + self.key = key + self.is_valid = is_valid + + +class RecentlyUsedContainer(MutableMapping): + """ + Provides a dict-like that maintains up to ``maxsize`` keys while throwing + away the least-recently-used keys beyond ``maxsize``. + """ + + # TODO: Make this threadsafe. _prune_invalidated_entries should be the + # only real pain-point for this. + + # If len(self.access_log) exceeds self._maxsize * CLEANUP_FACTOR, then we + # will attempt to cleanup the invalidated entries in the access_log + # datastructure during the next 'get' operation. + CLEANUP_FACTOR = 10 + + def __init__(self, maxsize=10): + self._maxsize = maxsize + + self._container = {} + + # We use a deque to to store our keys ordered by the last access. + self.access_log = deque() + + # We look up the access log entry by the key to invalidate it so we can + # insert a new authorative entry at the head without having to dig and + # find the old entry for removal immediately. + self.access_lookup = {} + + # Trigger a heap cleanup when we get past this size + self.access_log_limit = maxsize * self.CLEANUP_FACTOR + + def _push_entry(self, key): + "Push entry onto our access log, invalidate the old entry if exists." + # Invalidate old entry if it exists + old_entry = self.access_lookup.get(key) + if old_entry: + old_entry.is_valid = False + + new_entry = AccessEntry(key) + + self.access_lookup[key] = new_entry + self.access_log.appendleft(new_entry) + + def _prune_entries(self, num): + "Pop entries from our access log until we popped ``num`` valid ones." + while num > 0: + p = self.access_log.pop() + + if not p.is_valid: + continue # Invalidated entry, skip + + del self._container[p.key] + del self.access_lookup[p.key] + num -= 1 + + def _prune_invalidated_entries(self): + "Rebuild our access_log without the invalidated entries." + self.access_log = deque(e for e in self.access_log if e.is_valid) + + def _get_ordered_access_keys(self): + # Used for testing + return [e.key for e in self.access_log if e.is_valid] + + def __getitem__(self, key): + item = self._container.get(key) + + if not item: + return + + # Insert new entry with new high priority, also implicitly invalidates + # the old entry. + self._push_entry(key) + + if len(self.access_log) > self.access_log_limit: + # Heap is getting too big, try to clean up any tailing invalidated + # entries. + self._prune_invalidated_entries() + + return item + + def __setitem__(self, key, item): + # Add item to our container and access log + self._container[key] = item + self._push_entry(key) + + # Discard invalid and excess entries + self._prune_entries(len(self._container) - self._maxsize) + + def __delitem__(self, key): + self._invalidate_entry(key) + del self._container[key] + del self._access_lookup[key] + + def __len__(self): + return self._container.__len__() + + def __iter__(self): + return self._container.__iter__() + + def __contains__(self, key): + return self._container.__contains__(key) diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py new file mode 100644 index 00000000..4c2c6b54 --- /dev/null +++ b/requests/packages/urllib3/connectionpool.py @@ -0,0 +1,525 @@ +# urllib3/connectionpool.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +import logging +import socket + + +from httplib import HTTPConnection, HTTPSConnection, HTTPException +from Queue import Queue, Empty, Full +from select import select +from socket import error as SocketError, timeout as SocketTimeout + + +try: + import ssl + BaseSSLError = ssl.SSLError +except ImportError: + ssl = None + BaseSSLError = None + + +from .request import RequestMethods +from .response import HTTPResponse +from .exceptions import ( + SSLError, + MaxRetryError, + TimeoutError, + HostChangedError, + EmptyPoolError, +) + + +log = logging.getLogger(__name__) + +_Default = object() + + +## Connection objects (extension of httplib) + +class VerifiedHTTPSConnection(HTTPSConnection): + """ + Based on httplib.HTTPSConnection but wraps the socket with + SSL certification. + """ + cert_reqs = None + ca_certs = None + + def set_cert(self, key_file=None, cert_file=None, + cert_reqs='CERT_NONE', ca_certs=None): + ssl_req_scheme = { + 'CERT_NONE': ssl.CERT_NONE, + 'CERT_OPTIONAL': ssl.CERT_OPTIONAL, + 'CERT_REQUIRED': ssl.CERT_REQUIRED + } + + self.key_file = key_file + self.cert_file = cert_file + self.cert_reqs = ssl_req_scheme.get(cert_reqs) or ssl.CERT_NONE + self.ca_certs = ca_certs + + def connect(self): + # Add certificate verification + sock = socket.create_connection((self.host, self.port), self.timeout) + + # Wrap socket using verification with the root certs in + # trusted_root_certs + self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, + cert_reqs=self.cert_reqs, + ca_certs=self.ca_certs) + + +## Pool objects + +class ConnectionPool(object): + """ + Base class for all connection pools, such as + :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`. + """ + pass + + +class HTTPConnectionPool(ConnectionPool, RequestMethods): + """ + Thread-safe connection pool for one host. + + :param host: + Host used for this HTTP Connection (e.g. "localhost"), passed into + :class:`httplib.HTTPConnection`. + + :param port: + Port used for this HTTP Connection (None is equivalent to 80), passed + into :class:`httplib.HTTPConnection`. + + :param strict: + Causes BadStatusLine to be raised if the status line can't be parsed + as a valid HTTP/1.0 or 1.1 status line, passed into + :class:`httplib.HTTPConnection`. + + :param timeout: + Socket timeout for each individual connection, can be a float. None + disables timeout. + + :param maxsize: + Number of connections to save that can be reused. More than 1 is useful + in multithreaded situations. If ``block`` is set to false, more + connections will be created but they will not be saved once they've + been used. + + :param block: + If set to True, no more than ``maxsize`` connections will be used at + a time. When no free connections are available, the call will block + until a connection has been released. This is a useful side effect for + particular multithreaded situations where one does not want to use more + than maxsize connections per host to prevent flooding. + + :param headers: + Headers to include with all requests, unless other headers are given + explicitly. + """ + + scheme = 'http' + + def __init__(self, host, port=None, strict=False, timeout=None, maxsize=1, + block=False, headers=None): + self.host = host + self.port = port + self.strict = strict + self.timeout = timeout + self.pool = Queue(maxsize) + self.block = block + self.headers = headers or {} + + # Fill the queue up so that doing get() on it will block properly + for _ in xrange(maxsize): + self.pool.put(None) + + # These are mostly for testing and debugging purposes. + self.num_connections = 0 + self.num_requests = 0 + + def _new_conn(self): + """ + Return a fresh :class:`httplib.HTTPConnection`. + """ + self.num_connections += 1 + log.info("Starting new HTTP connection (%d): %s" % + (self.num_connections, self.host)) + return HTTPConnection(host=self.host, port=self.port) + + def _get_conn(self, timeout=None): + """ + Get a connection. Will return a pooled connection if one is available. + + If no connections are available and :prop:`.block` is ``False``, then a + fresh connection is returned. + + :param timeout: + Seconds to wait before giving up and raising + :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and + :prop:`.block` is ``True``. + """ + conn = None + try: + conn = self.pool.get(block=self.block, timeout=timeout) + + # If this is a persistent connection, check if it got disconnected + if conn and conn.sock and select([conn.sock], [], [], 0.0)[0]: + # Either data is buffered (bad), or the connection is dropped. + log.info("Resetting dropped connection: %s" % self.host) + conn.close() + + except Empty: + if self.block: + raise EmptyPoolError("Pool reached maximum size and no more " + "connections are allowed.") + pass # Oh well, we'll create a new connection then + + return conn or self._new_conn() + + def _put_conn(self, conn): + """ + Put a connection back into the pool. + + :param conn: + Connection object for the current host and port as returned by + :meth:`._new_conn` or :meth:`._get_conn`. + + If the pool is already full, the connection is discarded because we + exceeded maxsize. If connections are discarded frequently, then maxsize + should be increased. + """ + try: + self.pool.put(conn, block=False) + except Full: + # This should never happen if self.block == True + log.warning("HttpConnectionPool is full, discarding connection: %s" + % self.host) + + def _make_request(self, conn, method, url, timeout=_Default, + **httplib_request_kw): + """ + Perform a request on a given httplib connection object taken from our + pool. + """ + self.num_requests += 1 + + if timeout is _Default: + timeout = self.timeout + + conn.request(method, url, **httplib_request_kw) + conn.sock.settimeout(timeout) + httplib_response = conn.getresponse() + + log.debug("\"%s %s %s\" %s %s" % + (method, url, + conn._http_vsn_str, # pylint: disable-msg=W0212 + httplib_response.status, httplib_response.length)) + + return httplib_response + + + def is_same_host(self, url): + """ + Check if the given ``url`` is a member of the same host as this + conncetion pool. + """ + # TODO: Add optional support for socket.gethostbyname checking. + return (url.startswith('/') or + get_host(url) == (self.scheme, self.host, self.port)) + + def urlopen(self, method, url, body=None, headers=None, retries=3, + redirect=True, assert_same_host=True, timeout=_Default, + pool_timeout=None, release_conn=None, **response_kw): + """ + Get a connection from the pool and perform an HTTP request. This is the + lowest level call for making a request, so you'll need to specify all + the raw details. + + .. note:: + + More commonly, it's appropriate to use a convenience method provided + by :class:`.RequestMethods`, such as :meth:`.request`. + + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param body: + Data to send in the request body (useful for creating + POST requests, see HTTPConnectionPool.post_url for + more convenience). + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. If None, pool headers are used. If provided, + these headers completely replace any pool-specific headers. + + :param retries: + Number of retries to allow before raising a MaxRetryError exception. + + :param redirect: + Automatically handle redirects (status codes 301, 302, 303, 307), + each redirect counts as a retry. + + :param assert_same_host: + If ``True``, will make sure that the host of the pool requests is + consistent else will raise HostChangedError. When False, you can + use the pool on an HTTP proxy and request foreign hosts. + + :param timeout: + If specified, overrides the default timeout for this one request. + + :param pool_timeout: + If set and the pool is set to block=True, then this method will + block for ``pool_timeout`` seconds and raise EmptyPoolError if no + connection is available within the time period. + + :param release_conn: + If False, then the urlopen call will not release the connection + back into the pool once a response is received. This is useful if + you're not preloading the response's content immediately. You will + need to call ``r.release_conn()`` on the response ``r`` to return + the connection back into the pool. If None, it takes the value of + ``response_kw.get('preload_content', True)``. + + :param \**response_kw: + Additional parameters are passed to + :meth:`urllib3.response.HTTPResponse.from_httplib` + """ + if headers is None: + headers = self.headers + + if retries < 0: + raise MaxRetryError("Max retries exceeded for url: %s" % url) + + if release_conn is None: + release_conn = response_kw.get('preload_content', True) + + # Check host + if assert_same_host and not self.is_same_host(url): + host = "%s://%s" % (self.scheme, self.host) + if self.port: + host = "%s:%d" % (host, self.port) + + raise HostChangedError("Connection pool with host '%s' tried to " + "open a foreign host: %s" % (host, url)) + + conn = None + + try: + # Request a connection from the queue + # (Could raise SocketError: Bad file descriptor) + conn = self._get_conn(timeout=pool_timeout) + + # Make the request on the httplib connection object + httplib_response = self._make_request(conn, method, url, + timeout=timeout, + body=body, headers=headers) + + # If we're going to release the connection in ``finally:``, then + # the request doesn't need to know about the connection. Otherwise + # it will also try to release it and we'll have a double-release + # mess. + response_conn = not release_conn and conn + + # Import httplib's response into our own wrapper object + response = HTTPResponse.from_httplib(httplib_response, + pool=self, + connection=response_conn, + **response_kw) + + # else: + # The connection will be put back into the pool when + # ``response.release_conn()`` is called (implicitly by + # ``response.read()``) + + except (SocketTimeout, Empty), e: + # Timed out either by socket or queue + raise TimeoutError("Request timed out after %f seconds" % + self.timeout) + + except (BaseSSLError), e: + # SSL certificate error + raise SSLError(e) + + except (HTTPException, SocketError), e: + # Connection broken, discard. It will be replaced next _get_conn(). + conn = None + + finally: + if conn and release_conn: + # Put the connection back to be reused + self._put_conn(conn) + + if not conn: + log.warn("Retrying (%d attempts remain) after connection " + "broken by '%r': %s" % (retries, e, url)) + return self.urlopen(method, url, body, headers, retries - 1, + redirect, assert_same_host) # Try again + + # Handle redirection + if (redirect and + response.status in [301, 302, 303, 307] and + 'location' in response.headers): # Redirect, retry + log.info("Redirecting %s -> %s" % + (url, response.headers.get('location'))) + return self.urlopen(method, response.headers.get('location'), body, + headers, retries - 1, redirect, + assert_same_host) + + return response + + +class HTTPSConnectionPool(HTTPConnectionPool): + """ + Same as :class:`.HTTPConnectionPool`, but HTTPS. + + When Python is compiled with the :mod:`ssl` module, then + :class:`.VerifiedHTTPSConnection` is used, which *can* verify certificates, + instead of :class:httplib.HTTPSConnection`. + + The ``key_file``, ``cert_file``, ``cert_reqs``, and ``ca_certs`` parameters + are only used if :mod:`ssl` is available and are fed into + :meth:`ssl.wrap_socket` to upgrade the connection socket into an SSL socket. + """ + + scheme = 'https' + + def __init__(self, host, port=None, + strict=False, timeout=None, maxsize=1, + block=False, headers=None, + key_file=None, cert_file=None, + cert_reqs='CERT_NONE', ca_certs=None): + + super(HTTPSConnectionPool, self).__init__(host, port, + strict, timeout, maxsize, + block, headers) + self.key_file = key_file + self.cert_file = cert_file + self.cert_reqs = cert_reqs + self.ca_certs = ca_certs + + def _new_conn(self): + """ + Return a fresh :class:`httplib.HTTPSConnection`. + """ + self.num_connections += 1 + log.info("Starting new HTTPS connection (%d): %s" + % (self.num_connections, self.host)) + + if not ssl: + return HTTPSConnection(host=self.host, port=self.port) + + connection = VerifiedHTTPSConnection(host=self.host, port=self.port) + connection.set_cert(key_file=self.key_file, cert_file=self.cert_file, + cert_reqs=self.cert_reqs, ca_certs=self.ca_certs) + return connection + + +## Helpers + +def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, + basic_auth=None): + """ + Shortcuts for generating request headers. + + :param keep_alive: + If ``True``, adds 'connection: keep-alive' header. + + :param accept_encoding: + Can be a boolean, list, or string. + ``True`` translates to 'gzip,deflate'. + List will get joined by comma. + String will be used as provided. + + :param user_agent: + String representing the user-agent you want, such as + "python-urllib3/0.6" + + :param basic_auth: + Colon-separated username:password string for 'authorization: basic ...' + auth header. + + Example: :: + + >>> make_headers(keep_alive=True, user_agent="Batman/1.0") + {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} + >>> make_headers(accept_encoding=True) + {'accept-encoding': 'gzip,deflate'} + """ + headers = {} + if accept_encoding: + if isinstance(accept_encoding, str): + pass + elif isinstance(accept_encoding, list): + accept_encoding = ','.join(accept_encoding) + else: + accept_encoding = 'gzip,deflate' + headers['accept-encoding'] = accept_encoding + + if user_agent: + headers['user-agent'] = user_agent + + if keep_alive: + headers['connection'] = 'keep-alive' + + if basic_auth: + headers['authorization'] = 'Basic ' + \ + basic_auth.encode('base64').strip() + + return headers + + +def get_host(url): + """ + Given a url, return its scheme, host and port (None if it's not there). + + For example: :: + + >>> get_host('http://google.com/mail/') + ('http', 'google.com', None) + >>> get_host('google.com:80') + ('http', 'google.com', 80) + """ + # This code is actually similar to urlparse.urlsplit, but much + # simplified for our needs. + port = None + scheme = 'http' + if '//' in url: + scheme, url = url.split('://', 1) + if '/' in url: + url, _path = url.split('/', 1) + if ':' in url: + url, port = url.split(':', 1) + port = int(port) + return scheme, url, port + + +def connection_from_url(url, **kw): + """ + Given a url, return an :class:`.ConnectionPool` instance of its host. + + This is a shortcut for not having to parse out the scheme, host, and port + of the url before creating an :class:`.ConnectionPool` instance. + + :param url: + Absolute URL string that must include the scheme. Port is optional. + + :param \**kw: + Passes additional parameters to the constructor of the appropriate + :class:`.ConnectionPool`. Useful for specifying things like + timeout, maxsize, headers, etc. + + Example: :: + + >>> conn = connection_from_url('http://google.com/') + >>> r = conn.request('GET', '/') + """ + scheme, host, port = get_host(url) + if scheme == 'https': + return HTTPSConnectionPool(host, port=port, **kw) + else: + return HTTPConnectionPool(host, port=port, **kw) diff --git a/requests/packages/urllib3/contrib/__init__.py b/requests/packages/urllib3/contrib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/requests/packages/urllib3/contrib/ntlmpool.py b/requests/packages/urllib3/contrib/ntlmpool.py new file mode 100644 index 00000000..c5f010e1 --- /dev/null +++ b/requests/packages/urllib3/contrib/ntlmpool.py @@ -0,0 +1,117 @@ +# urllib3/contrib/ntlmpool.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +""" +NTLM authenticating pool, contributed by erikcederstran + +Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10 +""" + +import httplib +from logging import getLogger +from ntlm import ntlm + +from urllib3 import HTTPSConnectionPool + + +log = getLogger(__name__) + + +class NTLMConnectionPool(HTTPSConnectionPool): + """ + Implements an NTLM authentication version of an urllib3 connection pool + """ + + scheme = 'https' + + def __init__(self, user, pw, authurl, *args, **kwargs): + """ + authurl is a random URL on the server that is protected by NTLM. + user is the Windows user, probably in the DOMAIN\username format. + pw is the password for the user. + """ + super(NTLMConnectionPool, self).__init__(*args, **kwargs) + self.authurl = authurl + self.rawuser = user + user_parts = user.split('\\', 1) + self.domain = user_parts[0].upper() + self.user = user_parts[1] + self.pw = pw + + def _new_conn(self): + # Performs the NTLM handshake that secures the connection. The socket + # must be kept open while requests are performed. + self.num_connections += 1 + log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s' % + (self.num_connections, self.host, self.authurl)) + + headers = {} + headers['Connection'] = 'Keep-Alive' + req_header = 'Authorization' + resp_header = 'www-authenticate' + + conn = httplib.HTTPSConnection(host=self.host, port=self.port) + + # Send negotiation message + headers[req_header] = ( + 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)) + log.debug('Request headers: %s' % headers) + conn.request('GET', self.authurl, None, headers) + res = conn.getresponse() + reshdr = dict(res.getheaders()) + log.debug('Response status: %s %s' % (res.status, res.reason)) + log.debug('Response headers: %s' % reshdr) + log.debug('Response data: %s [...]' % res.read(100)) + + # Remove the reference to the socket, so that it can not be closed by + # the response object (we want to keep the socket open) + res.fp = None + + # Server should respond with a challenge message + auth_header_values = reshdr[resp_header].split(', ') + auth_header_value = None + for s in auth_header_values: + if s[:5] == 'NTLM ': + auth_header_value = s[5:] + if auth_header_value is None: + raise Exception('Unexpected %s response header: %s' % + (resp_header, reshdr[resp_header])) + + # Send authentication message + ServerChallenge, NegotiateFlags = \ + ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value) + auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, + self.user, + self.domain, + self.pw, + NegotiateFlags) + headers[req_header] = 'NTLM %s' % auth_msg + log.debug('Request headers: %s' % headers) + conn.request('GET', self.authurl, None, headers) + res = conn.getresponse() + log.debug('Response status: %s %s' % (res.status, res.reason)) + log.debug('Response headers: %s' % dict(res.getheaders())) + log.debug('Response data: %s [...]' % res.read()[:100]) + if res.status != 200: + if res.status == 401: + raise Exception('Server rejected request: wrong ' + 'username or password') + raise Exception('Wrong server response: %s %s' % + (res.status, res.reason)) + + res.fp = None + log.debug('Connection established') + return conn + + def urlopen(self, method, url, body=None, headers=None, retries=3, + redirect=True, assert_same_host=True): + if headers is None: + headers = {} + headers['Connection'] = 'Keep-Alive' + return super(NTLMConnectionPool, self).urlopen(method, url, body, + headers, retries, + redirect, + assert_same_host) diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py new file mode 100644 index 00000000..69f459bd --- /dev/null +++ b/requests/packages/urllib3/exceptions.py @@ -0,0 +1,35 @@ +# urllib3/exceptions.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +## Exceptions + +class HTTPError(Exception): + "Base exception used by this module." + pass + + +class SSLError(Exception): + "Raised when SSL certificate fails in an HTTPS connection." + pass + + +class MaxRetryError(HTTPError): + "Raised when the maximum number of retries is exceeded." + pass + + +class TimeoutError(HTTPError): + "Raised when a socket timeout occurs." + pass + + +class HostChangedError(HTTPError): + "Raised when an existing pool gets a request for a foreign host." + pass + +class EmptyPoolError(HTTPError): + "Raised when a pool runs out of connections and no more are allowed." + pass diff --git a/requests/packages/urllib3/filepost.py b/requests/packages/urllib3/filepost.py new file mode 100644 index 00000000..2ffea8bb --- /dev/null +++ b/requests/packages/urllib3/filepost.py @@ -0,0 +1,71 @@ +# urllib3/filepost.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +import codecs +import mimetools +import mimetypes + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO # pylint: disable-msg=W0404 + + +writer = codecs.lookup('utf-8')[3] + + +def get_content_type(filename): + return mimetypes.guess_type(filename)[0] or 'application/octet-stream' + + +def encode_multipart_formdata(fields, boundary=None): + """ + Encode a dictionary of ``fields`` using the multipart/form-data mime format. + + :param fields: + Dictionary of fields. The key is treated as the field name, and the + value as the body of the form-data. If the value is a tuple of two + elements, then the first element is treated as the filename of the + form-data section. + + :param boundary: + If not specified, then a random boundary will be generated using + :func:`mimetools.choose_boundary`. + """ + body = StringIO() + if boundary is None: + boundary = mimetools.choose_boundary() + + for fieldname, value in fields.iteritems(): + body.write('--%s\r\n' % (boundary)) + + if isinstance(value, tuple): + filename, data = value + writer(body).write('Content-Disposition: form-data; name="%s"; ' + 'filename="%s"\r\n' % (fieldname, filename)) + body.write('Content-Type: %s\r\n\r\n' % + (get_content_type(filename))) + else: + data = value + writer(body).write('Content-Disposition: form-data; name="%s"\r\n' + % (fieldname)) + body.write('Content-Type: text/plain\r\n\r\n') + + if isinstance(data, int): + data = str(data) # Backwards compatibility + + if isinstance(data, unicode): + writer(body).write(data) + else: + body.write(data) + + body.write('\r\n') + + body.write('--%s--\r\n' % (boundary)) + + content_type = 'multipart/form-data; boundary=%s' % boundary + + return body.getvalue(), content_type diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py new file mode 100644 index 00000000..622789e5 --- /dev/null +++ b/requests/packages/urllib3/poolmanager.py @@ -0,0 +1,128 @@ +# urllib3/poolmanager.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +from ._collections import RecentlyUsedContainer +from .connectionpool import ( + HTTPConnectionPool, HTTPSConnectionPool, + get_host, connection_from_url, +) + + +__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] + + +from .request import RequestMethods +from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool + + +pool_classes_by_scheme = { + 'http': HTTPConnectionPool, + 'https': HTTPSConnectionPool, +} + +port_by_scheme = { + 'http': 80, + 'https': 443, +} + + +class PoolManager(RequestMethods): + """ + Allows for arbitrary requests while transparently keeping track of + necessary connection pools for you. + + :param num_pools: + Number of connection pools to cache before discarding the least recently + used pool. + + :param \**connection_pool_kw: + Additional parameters are used to create fresh + :class:`urllib3.connectionpool.ConnectionPool` instances. + + Example: :: + + >>> manager = PoolManager() + >>> r = manager.urlopen("http://google.com/") + >>> r = manager.urlopen("http://google.com/mail") + >>> r = manager.urlopen("http://yahoo.com/") + >>> len(r.pools) + 2 + + """ + + # TODO: Make sure there are no memory leaks here. + + def __init__(self, num_pools=10, **connection_pool_kw): + self.connection_pool_kw = connection_pool_kw + self.pools = RecentlyUsedContainer(num_pools) + + def connection_from_host(self, host, port=80, scheme='http'): + """ + Get a :class:`ConnectionPool` based on the host, port, and scheme. + + Note that an appropriate ``port`` value is required here to normalize + connection pools in our container most effectively. + """ + pool_key = (scheme, host, port) + + # If the scheme, host, or port doesn't match existing open connections, + # open a new ConnectionPool. + pool = self.pools.get(pool_key) + if pool: + return pool + + # Make a fresh ConnectionPool of the desired type + pool_cls = pool_classes_by_scheme[scheme] + pool = pool_cls(host, port, **self.connection_pool_kw) + + self.pools[pool_key] = pool + + return pool + + def connection_from_url(self, url): + """ + Similar to :func:`urllib3.connectionpool.connection_from_url` but + doesn't pass any additional parameters to the + :class:`urllib3.connectionpool.ConnectionPool` constructor. + + Additional parameters are taken from the :class:`.PoolManager` + constructor. + """ + scheme, host, port = get_host(url) + + port = port or port_by_scheme.get(scheme, 80) + + return self.connection_from_host(host, port=port, scheme=scheme) + + def urlopen(self, method, url, **kw): + """ + Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`. + + ``url`` must be absolute, such that an appropriate + :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it. + """ + conn = self.connection_from_url(url) + return conn.urlopen(method, url, assert_same_host=False, **kw) + + +class ProxyManager(object): + """ + Given a ConnectionPool to a proxy, the ProxyManager's ``urlopen`` method + will make requests to any url through the defined proxy. + """ + + def __init__(self, proxy_pool): + self.proxy_pool = proxy_pool + + def urlopen(self, method, url, **kw): + "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." + kw['assert_same_host'] = False + return self.proxy_pool.urlopen(method, url, **kw) + + +def proxy_from_url(url, **pool_kw): + proxy_pool = connection_from_url(url, **pool_kw) + return ProxyManager(proxy_pool) diff --git a/requests/packages/urllib3/request.py b/requests/packages/urllib3/request.py new file mode 100644 index 00000000..a7e0b5de --- /dev/null +++ b/requests/packages/urllib3/request.py @@ -0,0 +1,145 @@ +# urllib3/request.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + + +from urllib import urlencode + +from .filepost import encode_multipart_formdata + + +__all__ = ['RequestMethods'] + + +class RequestMethods(object): + """ + Convenience mixin for classes who implement a :meth:`urlopen` method, such + as :class:`~urllib3.connectionpool.HTTPConnectionPool` and + :class:`~urllib3.poolmanager.PoolManager`. + + Provides behavior for making common types of HTTP request methods and + decides which type of request field encoding to use. + + Specifically, + + :meth:`.request_encode_url` is for sending requests whose fields are encoded + in the URL (such as GET, HEAD, DELETE). + + :meth:`.request_encode_body` is for sending requests whose fields are + encoded in the *body* of the request using multipart or www-orm-urlencoded + (such as for POST, PUT, PATCH). + + :meth:`.request` is for making any kind of request, it will look up the + appropriate encoding format and use one of the above two methods to make + the request. + """ + + _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS']) + + _encode_body_methods = set(['PATCH', 'POST', 'PUT', 'TRACE']) + + def urlopen(self, method, url, body=None, headers=None, + encode_multipart=True, multipart_boundary=None, + **kw): + raise NotImplemented("Classes extending RequestMethods must implement " + "their own ``urlopen`` method.") + + def request(self, method, url, fields=None, headers=None, **urlopen_kw): + """ + Make a request using :meth:`urlopen` with the appropriate encoding of + ``fields`` based on the ``method`` used. + + This is a convenience method that requires the least amount of manual + effort. It can be used in most situations, while still having the option + to drop down to more specific methods when necessary, such as + :meth:`request_encode_url`, :meth:`request_encode_body`, + or even the lowest level :meth:`urlopen`. + """ + method = method.upper() + + if method in self._encode_url_methods: + return self.request_encode_url(method, url, fields=fields, + headers=headers, + **urlopen_kw) + else: + return self.request_encode_body(method, url, fields=fields, + headers=headers, + **urlopen_kw) + + def request_encode_url(self, method, url, fields=None, **urlopen_kw): + """ + Make a request using :meth:`urlopen` with the ``fields`` encoded in + the url. This is useful for request methods like GET, HEAD, DELETE, etc. + """ + if fields: + url += '?' + urlencode(fields) + return self.urlopen(method, url, **urlopen_kw) + + def request_encode_body(self, method, url, fields=None, headers=None, + encode_multipart=True, multipart_boundary=None, + **urlopen_kw): + """ + Make a request using :meth:`urlopen` with the ``fields`` encoded in + the body. This is useful for request methods like POST, PUT, PATCH, etc. + + When ``encode_multipart=True`` (default), then + :meth:`urllib3.filepost.encode_multipart_formdata` is used to encode the + payload with the appropriate content type. Otherwise + :meth:`urllib.urlencode` is used with the + 'application/x-www-form-urlencoded' content type. + + Multipart encoding must be used when posting files, and it's reasonably + safe to use it in other times too. However, it may break request signing, + such as with OAuth. + + Supports an optional ``fields`` parameter of key/value strings AND + key/filetuple. A filetuple is a (filename, data) tuple. For example: :: + + fields = { + 'foo': 'bar', + 'fakefile': ('foofile.txt', 'contents of foofile'), + 'realfile': ('barfile.txt', open('realfile').read()), + 'nonamefile': ('contents of nonamefile field'), + } + + When uploading a file, providing a filename (the first parameter of the + tuple) is optional but recommended to best mimick behavior of browsers. + + Note that if ``headers`` are supplied, the 'Content-Type' header will be + overwritten because it depends on the dynamic random boundary string + which is used to compose the body of the request. The random boundary + string can be explicitly set with the ``multipart_boundary`` parameter. + """ + if encode_multipart: + body, content_type = encode_multipart_formdata(fields or {}, + boundary=multipart_boundary) + else: + body, content_type = (urlencode(fields or {}), + 'application/x-www-form-urlencoded') + + headers = headers or {} + headers.update({'Content-Type': content_type}) + + return self.urlopen(method, url, body=body, headers=headers, + **urlopen_kw) + + # Deprecated: + + def get_url(self, url, fields=None, **urlopen_kw): + """ + .. deprecated:: 1.0 + Use :meth:`request` instead. + """ + return self.request_encode_url('GET', url, fields=fields, + **urlopen_kw) + + def post_url(self, url, fields=None, headers=None, **urlopen_kw): + """ + .. deprecated:: 1.0 + Use :meth:`request` instead. + """ + return self.request_encode_body('POST', url, fields=fields, + headers=headers, + **urlopen_kw) diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py new file mode 100644 index 00000000..4cd15c11 --- /dev/null +++ b/requests/packages/urllib3/response.py @@ -0,0 +1,181 @@ +# urllib3/response.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + +import gzip +import logging +import zlib + + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO # pylint: disable-msg=W0404 + + +from .exceptions import HTTPError + + +log = logging.getLogger(__name__) + + +def decode_gzip(data): + gzipper = gzip.GzipFile(fileobj=StringIO(data)) + return gzipper.read() + + +def decode_deflate(data): + try: + return zlib.decompress(data) + except zlib.error: + return zlib.decompress(data, -zlib.MAX_WBITS) + + +class HTTPResponse(object): + """ + HTTP Response container. + + Backwards-compatible to httplib's HTTPResponse but the response ``body`` is + loaded and decoded on-demand when the ``data`` property is accessed. + + Extra parameters for behaviour not present in httplib.HTTPResponse: + + :param preload_content: + If True, the response's body will be preloaded during construction. + + :param decode_content: + If True, attempts to decode specific content-encoding's based on headers + (like 'gzip' and 'deflate') will be skipped and raw data will be used + instead. + + :param original_response: + When this HTTPResponse wrapper is generated from an httplib.HTTPResponse + object, it's convenient to include the original for debug purposes. It's + otherwise unused. + """ + + CONTENT_DECODERS = { + 'gzip': decode_gzip, + 'deflate': decode_deflate, + } + + def __init__(self, body='', headers=None, status=0, version=0, reason=None, + strict=0, preload_content=True, decode_content=True, + original_response=None, pool=None, connection=None): + self.headers = headers or {} + self.status = status + self.version = version + self.reason = reason + self.strict = strict + + self._decode_content = decode_content + self._body = None + self._fp = None + self._original_response = original_response + + self._pool = pool + self._connection = connection + + if hasattr(body, 'read'): + self._fp = body + + if preload_content: + self._body = self.read(decode_content=decode_content) + + def release_conn(self): + if not self._pool or not self._connection: + return + + self._pool._put_conn(self._connection) + self._connection = None + + @property + def data(self): + # For backwords-compat with earlier urllib3 0.4 and earlier. + if self._body: + return self._body + + if self._fp: + return self.read(decode_content=self._decode_content, + cache_content=True) + + def read(self, amt=None, decode_content=True, cache_content=False): + """ + Similar to :meth:`httplib.HTTPResponse.read`, but with two additional + parameters: ``decode_content`` and ``cache_content``. + + :param amt: + How much of the content to read. If specified, decoding and caching + is skipped because we can't decode partial content nor does it make + sense to cache partial content as the full response. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. (Overridden if ``amt`` is set.) + + :param cache_content: + If True, will save the returned data such that the same result is + returned despite of the state of the underlying file object. This + is useful if you want the ``.data`` property to continue working + after having ``.read()`` the file object. (Overridden if ``amt`` is + set.) + """ + content_encoding = self.headers.get('content-encoding') + decoder = self.CONTENT_DECODERS.get(content_encoding) + + data = self._fp and self._fp.read(amt) + + try: + + if amt: + return data + + if not decode_content or not decoder: + if cache_content: + self._body = data + + return data + + try: + data = decoder(data) + except IOError: + raise HTTPError("Received response with content-encoding: %s, but " + "failed to decode it." % content_encoding) + + if cache_content: + self._body = data + + return data + + finally: + + if self._original_response and self._original_response.isclosed(): + self.release_conn() + + @staticmethod + def from_httplib(r, **response_kw): + """ + Given an :class:`httplib.HTTPResponse` instance ``r``, return a + corresponding :class:`urllib3.response.HTTPResponse` object. + + Remaining parameters are passed to the HTTPResponse constructor, along + with ``original_response=r``. + """ + + return HTTPResponse(body=r, + headers=dict(r.getheaders()), + status=r.status, + version=r.version, + reason=r.reason, + strict=r.strict, + original_response=r, + **response_kw) + + # Backwards-compatibility methods for httplib.HTTPResponse + def getheaders(self): + return self.headers + + def getheader(self, name, default=None): + return self.headers.get(name, default) From dac050bf403749524e025b1cae697a61dc592217 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 16:36:53 -0400 Subject: [PATCH 160/255] New status_code-based errors --- requests/models.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 1fac1a8d..a77e8719 100644 --- a/requests/models.py +++ b/requests/models.py @@ -22,7 +22,7 @@ from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers from .utils import (dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip) from .status_codes import codes -from .exceptions import Timeout, URLRequired, TooManyRedirects +from .exceptions import Timeout, URLRequired, TooManyRedirects, RequestException from .monkeys import Request as _Request from .monkeys import HTTPRedirectHandler @@ -514,6 +514,17 @@ class Response(object): def raise_for_status(self): """Raises stored :class:`HTTPError` or :class:`URLError`, if one occurred.""" + if self.error: raise self.error + if (self.status_code >= 300) and (self.status_code < 400): + raise RequestException('%s Redirection' % self.status_code) + + elif (self.status_code >= 400) and (self.status_code < 500): + raise RequestException('%s Client Error' % self.status_code) + + elif (self.status_code >= 500) and (self.status_code < 600): + raise RequestException('%s Server Error' % self.status_code) + + From 28a9534383ed57c799a58ba57e53e0988ac385aa Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 18:15:02 -0400 Subject: [PATCH 161/255] add pool configurations to defaults --- requests/defaults.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requests/defaults.py b/requests/defaults.py index 951c056d..e3f5478d 100644 --- a/requests/defaults.py +++ b/requests/defaults.py @@ -35,4 +35,5 @@ defaults['max_redirects'] = 30 defaults['decode_unicode'] = True defaults['timeout_fallback'] = True # defaults['keep_alive'] = True -# defaults['max_connections'] = 10 \ No newline at end of file +defaults['pool_connections'] = 10 +defaults['pool_maxsize'] = 1 \ No newline at end of file From 50c39fd95ada7242fc6bb0cbfd68be1e4a32ac3b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 18:15:13 -0400 Subject: [PATCH 162/255] poolmanager in sessions --- requests/sessions.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/requests/sessions.py b/requests/sessions.py index 5993b27c..c90bd9aa 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -15,6 +15,7 @@ from .defaults import defaults from .models import Request from .hooks import dispatch_hook from .utils import add_dict_to_cookiejar, cookiejar_from_dict, header_expand +from .packages.urllib3.poolmanager import PoolManager def merge_kwargs(local_kwarg, default_kwarg): @@ -76,6 +77,11 @@ class Session(object): for (k, v) in defaults.items(): self.config.setdefault(k, v) + self.poolmanager = PoolManager( + num_pools=self.config.get('pool_connections'), + maxsize=self.config.get('pool_maxsize') + ) + # Set up a CookieJar to be used by default self.cookies = cookielib.FileCookieJar() @@ -148,7 +154,8 @@ class Session(object): timeout=timeout, allow_redirects=allow_redirects, proxies=proxies, - config=config + config=config, + _poolmanager=self.poolmanager ) for attr in self.__attrs__: From 3d3f05884f65efaa74e44e469a74873699df271e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 18:15:21 -0400 Subject: [PATCH 163/255] use new system --- requests/models.py | 189 +++++++++++++++++++++++++-------------------- 1 file changed, 104 insertions(+), 85 deletions(-) diff --git a/requests/models.py b/requests/models.py index a77e8719..ce7c55f5 100644 --- a/requests/models.py +++ b/requests/models.py @@ -8,9 +8,9 @@ requests.models """ import urllib -import urllib2 import socket import zlib +from Cookie import SimpleCookie from urllib2 import HTTPError from urlparse import urlparse, urlunparse, urljoin @@ -23,8 +23,6 @@ from .packages.poster.streaminghttp import register_openers, get_handlers from .utils import (dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip) from .status_codes import codes from .exceptions import Timeout, URLRequired, TooManyRedirects, RequestException -from .monkeys import Request as _Request -from .monkeys import HTTPRedirectHandler from .auth import dispatch as auth_dispatch @@ -51,7 +49,8 @@ class Request(object): allow_redirects=False, proxies=None, hooks=None, - config=None): + config=None, + _poolmanager=None): #: Float describes the timeout of the request. # (Use socket.setdefaulttimeout() as fallback) @@ -119,6 +118,7 @@ class Request(object): headers[k] = v self.headers = headers + self._poolmanager = _poolmanager # Pre-request hook. r = dispatch_hook('pre_request', hooks, self) @@ -129,37 +129,37 @@ class Request(object): return '' % (self.method) - def _get_opener(self): - """Creates appropriate opener object for urllib2.""" + # def _get_opener(self): + # """Creates appropriate opener object for urllib2.""" - _handlers = [] + # _handlers = [] - if self.cookies is not None: - _handlers.append(urllib2.HTTPCookieProcessor(self.cookies)) + # if self.cookies is not None: + # _handlers.append(urllib2.HTTPCookieProcessor(self.cookies)) - if self.proxies: - _handlers.append(urllib2.ProxyHandler(self.proxies)) + # if self.proxies: + # _handlers.append(urllib2.ProxyHandler(self.proxies)) - _handlers.append(HTTPRedirectHandler) + # _handlers.append(HTTPRedirectHandler) - if not _handlers: - return urllib2.urlopen + # if not _handlers: + # return urllib2.urlopen - if self.data or self.files: - _handlers.extend(get_handlers()) + # if self.data or self.files: + # _handlers.extend(get_handlers()) - opener = urllib2.build_opener(*_handlers) + # opener = urllib2.build_opener(*_handlers) - if self.headers: - # Allow default headers in the opener to be overloaded - normal_keys = [k.capitalize() for k in self.headers] - for key, val in opener.addheaders[:]: - if key not in normal_keys: - continue - # Remove it, we have a value to take its place - opener.addheaders.remove((key, val)) + # if self.headers: + # # Allow default headers in the opener to be overloaded + # normal_keys = [k.capitalize() for k in self.headers] + # for key, val in opener.addheaders[:]: + # if key not in normal_keys: + # continue + # # Remove it, we have a value to take its place + # opener.addheaders.remove((key, val)) - return opener.open + # return opener.open def _build_response(self, resp, is_error=False): @@ -171,24 +171,39 @@ class Request(object): def build(resp): response = Response() + + # Pass settings over. response.config = self.config - response.status_code = getattr(resp, 'code', None) - try: - response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None)) - response.raw = resp + # Fallback to None if there's no staus_code, for whatever reason. + response.status_code = getattr(resp, 'status', None) - if self.cookies: - response.cookies = dict_from_cookiejar(self.cookies) + # Make headers case-insensitive. + response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) + # Start off with our local cookies. + cookies = self.cookies or dict() - except AttributeError: - pass + # Add new cookies from the server. + if 'set-cookie' in response.headers: + cookie_header = response.headers['set-cookie'] + + c = SimpleCookie() + c.load(cookie_header) + + for k,v in c.items(): + cookies.update({k: v.value}) + + # Save cookies in Response. + response.cookies = cookies + + # Save original resopnse for later. + response.raw = resp if is_error: response.error = resp - response.url = getattr(resp, 'url', None) + response.url = self._build_url() return response @@ -204,7 +219,7 @@ class Request(object): ((r.status_code is codes.see_other) or (self.allow_redirects)) ): - r.raw.close() + # r.raw.close() if not len(history) < self.config.get('max_redirects'): raise TooManyRedirects() @@ -239,7 +254,8 @@ class Request(object): auth=self.auth, cookies=self.cookies, redirect=True, - config=self.config + config=self.config, + _poolmanager=self._poolmanager ) request.send() r = request.response @@ -247,6 +263,7 @@ class Request(object): r.history = history self.response = r + self.response.ok = True self.response.request = self @@ -317,19 +334,26 @@ class Request(object): # Build the URL url = self._build_url() - # Attach uploaded files. + # Nottin' on you. + body = None + content_type = None + + from .packages.urllib3.filepost import encode_multipart_formdata + + # Multi-part file uploads. if self.files: - register_openers() + if not isinstance(self.data, basestring): + fields = self.data.copy() + for (k, v) in self.files.items(): + fields.update({k: (k, v.read())}) + (body, content_type) = encode_multipart_formdata(fields) - # Add form-data to the multipart. - if self.data: - self.files.update(self.data) + # TODO: Setup cookies. - data, headers = multipart_encode(self.files) + # Add content-type if it wasn't explicitly provided. + if (content_type) and (not 'content-type' in self.headers): + self.headers['Content-Type'] = content_type - else: - data = self._enc_data - headers = {} if self.auth: auth_func, auth_args = self.auth @@ -338,65 +362,60 @@ class Request(object): self.__dict__.update(r.__dict__) - # Build the Urllib2 Request. - req = _Request(url, data=data, headers=headers, method=self.method) - # Add the headers to the request. - if self.headers: - for k,v in self.headers.iteritems(): - req.add_header(k, v) + conn = self._poolmanager.connection_from_url(url) if not self.sent or anyway: try: - opener = self._get_opener() - try: + if self.cookies: - resp = opener(req, timeout=self.timeout) + # Skip if 'cookie' header is explicitly set. + if 'cookie' not in self.headers: - except TypeError, err: - # timeout argument is new since Python v2.6 - if not 'timeout' in str(err): - raise + # Simple cookie with our dict. + # TODO: Multi-value headers. + c = SimpleCookie() + c.load(self.cookies) - if self.config.get('timeout_fallback'): - # fall-back and use global socket timeout (This is not thread-safe!) - old_timeout = socket.getdefaulttimeout() - socket.setdefaulttimeout(self.timeout) + # Turn it into a header. + cookie_header = c.output(header='').strip() - resp = opener(req) + # Attach Cookie header to request. + self.headers['Cookie'] = cookie_header - if self.config.get('timeout_fallback'): - # restore global timeout - socket.setdefaulttimeout(old_timeout) + # Create the connection. + r = conn.urlopen( + method=self.method, + url=url, + body=body, + headers=self.headers, + redirect=False, + assert_same_host=False, + preload_content=False, + decode_content=False + ) - if self.cookies is not None: - self.cookies.extract_cookies(resp, req) + # resp = {} - except (urllib2.HTTPError, urllib2.URLError), why: - if hasattr(why, 'reason'): - if isinstance(why.reason, socket.timeout): - why = Timeout(why) - elif isinstance(why.reason, socket.error): - why = Timeout(why) - self._build_response(why, is_error=True) + except ArithmeticError: + pass else: - self._build_response(resp) + self._build_response(r) self.response.ok = True + # self.sent = self.response.ok - self.sent = self.response.ok + # Response manipulation hook. + self.response = dispatch_hook('response', self.hooks, self.response) - # Response manipulation hook. - self.response = dispatch_hook('response', self.hooks, self.response) + # Post-request hook. + r = dispatch_hook('post_request', self.hooks, self) + self.__dict__.update(r.__dict__) - # Post-request hook. - r = dispatch_hook('post_request', self.hooks, self) - self.__dict__.update(r.__dict__) - - return self.sent + return self.sent class Response(object): From a9822b0bb6917cbc04d7c3d3d7f287b5961283eb Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 18:24:14 -0400 Subject: [PATCH 164/255] fix content uploads --- requests/models.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/requests/models.py b/requests/models.py index ce7c55f5..152ef854 100644 --- a/requests/models.py +++ b/requests/models.py @@ -347,6 +347,16 @@ class Request(object): for (k, v) in self.files.items(): fields.update({k: (k, v.read())}) (body, content_type) = encode_multipart_formdata(fields) + else: + pass + # TODO: Conflict? + else: + if self.data: + body = self._enc_data + content_type = 'application/x-www-form-urlencoded' + print body + + # TODO: Setup cookies. From a548b6249dda248a45f54ce5bf513f9c621d60c3 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 18:27:54 -0400 Subject: [PATCH 165/255] HTTPError --- requests/__init__.py | 2 +- requests/exceptions.py | 3 +++ requests/models.py | 9 +++------ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/requests/__init__.py b/requests/__init__.py index 5723dc44..68c7a089 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -29,5 +29,5 @@ from .sessions import session from .status_codes import codes from .exceptions import ( RequestException, AuthenticationError, Timeout, URLRequired, - TooManyRedirects + TooManyRedirects, HTTPError ) diff --git a/requests/exceptions.py b/requests/exceptions.py index 101d1ce6..16acd925 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -12,6 +12,9 @@ class RequestException(Exception): """There was an ambiguous exception that occurred while handling your request.""" +class HTTPError(RequestException): + """An HTTP error occured.""" + class AuthenticationError(RequestException): """The authentication credentials provided were invalid.""" diff --git a/requests/models.py b/requests/models.py index 152ef854..01a1329d 100644 --- a/requests/models.py +++ b/requests/models.py @@ -354,9 +354,6 @@ class Request(object): if self.data: body = self._enc_data content_type = 'application/x-www-form-urlencoded' - print body - - # TODO: Setup cookies. @@ -548,12 +545,12 @@ class Response(object): raise self.error if (self.status_code >= 300) and (self.status_code < 400): - raise RequestException('%s Redirection' % self.status_code) + raise HTTPError('%s Redirection' % self.status_code) elif (self.status_code >= 400) and (self.status_code < 500): - raise RequestException('%s Client Error' % self.status_code) + raise HTTPError('%s Client Error' % self.status_code) elif (self.status_code >= 500) and (self.status_code < 600): - raise RequestException('%s Server Error' % self.status_code) + raise HTTPError('%s Server Error' % self.status_code) From 4673d3dfa7c9f97a2b0950066ec53623fd0d0a41 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 18:28:32 -0400 Subject: [PATCH 166/255] v0.8.0 --- requests/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/__init__.py b/requests/__init__.py index 68c7a089..feb617e2 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -15,8 +15,8 @@ requests """ __title__ = 'requests' -__version__ = '0.7.2' -__build__ = 0x000702 +__version__ = '0.8.0' +__build__ = 0x000800 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' From d04f31b302cef097e061bcdf6e1a45d522c72abc Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 23 Oct 2011 18:33:27 -0400 Subject: [PATCH 167/255] HTTPError --- requests/models.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/requests/models.py b/requests/models.py index 01a1329d..f2bc74ef 100644 --- a/requests/models.py +++ b/requests/models.py @@ -12,7 +12,6 @@ import socket import zlib from Cookie import SimpleCookie -from urllib2 import HTTPError from urlparse import urlparse, urlunparse, urljoin from datetime import datetime @@ -22,7 +21,7 @@ from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers from .utils import (dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip) from .status_codes import codes -from .exceptions import Timeout, URLRequired, TooManyRedirects, RequestException +from .exceptions import Timeout, URLRequired, TooManyRedirects, RequestException, HTTPError from .auth import dispatch as auth_dispatch @@ -263,7 +262,13 @@ class Request(object): r.history = history self.response = r - self.response.ok = True + + try: + r.raise_for_status() + self.response.ok = True + except HTTPError: + self.response.ok = False + self.response.request = self @@ -475,11 +480,15 @@ class Response(object): def __repr__(self): return '' % (self.status_code) - def __nonzero__(self): """Returns true if :attr:`status_code` is 'OK'.""" - return not self.error + try: + self.raise_for_status() + except HTTPError: + return False + finally: + return True def iter_content(self, chunk_size=10 * 1024, decode_unicode=None): """Iterates over the response data. This avoids reading the content From 428fc9c0bd889eb673fcfe0803e7eab0e3712ea6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 24 Oct 2011 01:46:05 -0400 Subject: [PATCH 168/255] PASS ALL TEH TESTS --- requests/models.py | 108 +++++++++++++++++++++++---------------------- test_requests.py | 21 +-------- 2 files changed, 56 insertions(+), 73 deletions(-) diff --git a/requests/models.py b/requests/models.py index f2bc74ef..f86d8048 100644 --- a/requests/models.py +++ b/requests/models.py @@ -262,13 +262,6 @@ class Request(object): r.history = history self.response = r - - try: - r.raise_for_status() - self.response.ok = True - except HTTPError: - self.response.ok = False - self.response.request = self @@ -284,6 +277,9 @@ class Request(object): returns it twice. """ + if hasattr(data, '__iter__'): + data = dict(data) + if hasattr(data, 'items'): result = [] for k, vs in data.items(): @@ -298,11 +294,20 @@ class Request(object): def _build_url(self): """Build the actual URL to use.""" + if not self.url: + raise URLRequired() + # Support for unicode domain names and paths. scheme, netloc, path, params, query, fragment = urlparse(self.url) + + if not scheme: + raise ValueError() + netloc = netloc.encode('idna') + if isinstance(path, unicode): path = path.encode('utf-8') + path = urllib.quote(urllib.unquote(path)) self.url = str(urlunparse([ scheme, netloc, path, params, query, fragment ])) @@ -326,10 +331,6 @@ class Request(object): already been sent. """ - # Some people... - if not self.url: - raise URLRequired - # Logging if self.config.get('verbose'): self.config.get('verbose').write('%s %s %s\n' % ( @@ -348,7 +349,12 @@ class Request(object): # Multi-part file uploads. if self.files: if not isinstance(self.data, basestring): - fields = self.data.copy() + + try: + fields = self.data.copy() + except AttributeError: + fields = dict(self.data) + for (k, v) in self.files.items(): fields.update({k: (k, v.read())}) (body, content_type) = encode_multipart_formdata(fields) @@ -379,46 +385,35 @@ class Request(object): if not self.sent or anyway: - try: - if self.cookies: + if self.cookies: - # Skip if 'cookie' header is explicitly set. - if 'cookie' not in self.headers: + # Skip if 'cookie' header is explicitly set. + if 'cookie' not in self.headers: - # Simple cookie with our dict. - # TODO: Multi-value headers. - c = SimpleCookie() - c.load(self.cookies) + # Simple cookie with our dict. + # TODO: Multi-value headers. + c = SimpleCookie() + c.load(self.cookies) - # Turn it into a header. - cookie_header = c.output(header='').strip() + # Turn it into a header. + cookie_header = c.output(header='').strip() - # Attach Cookie header to request. - self.headers['Cookie'] = cookie_header + # Attach Cookie header to request. + self.headers['Cookie'] = cookie_header - # Create the connection. - r = conn.urlopen( - method=self.method, - url=url, - body=body, - headers=self.headers, - redirect=False, - assert_same_host=False, - preload_content=False, - decode_content=False - ) + # Create the connection. + r = conn.urlopen( + method=self.method, + url=url, + body=body, + headers=self.headers, + redirect=False, + assert_same_host=False, + preload_content=False, + decode_content=False + ) - # resp = {} - - - except ArithmeticError: - pass - - else: - self._build_response(r) - self.response.ok = True - - # self.sent = self.response.ok + self._build_response(r) # Response manipulation hook. self.response = dispatch_hook('response', self.hooks, self.response) @@ -456,9 +451,6 @@ class Response(object): #: Final URL location of Response. self.url = None - #: True if no :attr:`error` occurred. - self.ok = False - #: Resulting :class:`HTTPError` of request, if one occurred. self.error = None @@ -482,13 +474,16 @@ class Response(object): def __nonzero__(self): """Returns true if :attr:`status_code` is 'OK'.""" + return self.ok + @property + def ok(self): try: self.raise_for_status() except HTTPError: return False - finally: - return True + return True + def iter_content(self, chunk_size=10 * 1024, decode_unicode=None): """Iterates over the response data. This avoids reading the content @@ -497,8 +492,9 @@ class Response(object): length of each item returned as decoding can take place. """ if self._content_consumed: - raise RuntimeError('The content for this response was ' - 'already consumed') + raise RuntimeError( + 'The content for this response was already consumed' + ) def generate(): while 1: @@ -507,15 +503,21 @@ class Response(object): break yield chunk self._content_consumed = True + gen = generate() + if 'gzip' in self.headers.get('content-encoding', ''): gen = stream_decode_gzip(gen) + if decode_unicode is None: decode_unicode = self.config.get('decode_unicode') + if decode_unicode: gen = stream_decode_response_unicode(gen, self) + return gen + @property def content(self): """Content of the response, in bytes or unicode diff --git a/test_requests.py b/test_requests.py index 82f95954..fa9390f5 100755 --- a/test_requests.py +++ b/test_requests.py @@ -234,6 +234,7 @@ class RequestsTestSuite(unittest.TestCase): for service in SERVICES: r = requests.get(service('status', '404')) + print r.status_code self.assertEqual(r.ok, False) @@ -246,26 +247,6 @@ class RequestsTestSuite(unittest.TestCase): r.raise_for_status() - def test_cookie_jar(self): - - jar = cookielib.CookieJar() - self.assertFalse(jar) - - url = httpbin('cookies', 'set', 'requests_cookie', 'awesome') - r = requests.get(url, cookies=jar) - self.assertTrue(jar) - - cookie_found = False - for cookie in jar: - if cookie.name == 'requests_cookie': - self.assertEquals(cookie.value, 'awesome') - cookie_found = True - self.assertTrue(cookie_found) - - r = requests.get(httpbin('cookies'), cookies=jar) - self.assertTrue('awesome' in r.content) - - def test_decompress_gzip(self): r = requests.get(httpbin('gzip')) From b8c55da48d780c3688a5df724d87b8daa739b3f8 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 27 Oct 2011 01:08:51 -0400 Subject: [PATCH 169/255] stuff and things and stuff --- requests/models.py | 12 +++--------- requests/sessions.py | 11 ++--------- test_requests.py | 15 ++++++++++++--- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/requests/models.py b/requests/models.py index 35058911..bd3d9abc 100644 --- a/requests/models.py +++ b/requests/models.py @@ -18,16 +18,9 @@ from datetime import datetime from .auth import dispatch as auth_dispatch from .hooks import dispatch_hook from .structures import CaseInsensitiveDict -from .packages.poster.encode import multipart_encode -from .packages.poster.streaminghttp import register_openers, get_handlers from .status_codes import codes -from .exceptions import Timeout, URLRequired, TooManyRedirects -from .monkeys import Request as _Request -from .monkeys import HTTPRedirectHandler -from .exceptions import Timeout, URLRequired, TooManyRedirects, RequestException, HTTPError -from .exceptions import Timeout, URLRequired, TooManyRedirects -from .monkeys import Request as _Request -from .monkeys import HTTPRedirectHandler +from .exceptions import Timeout, URLRequired, TooManyRedirects, HTTPError + from .utils import ( dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip) @@ -190,6 +183,7 @@ class Request(object): # Start off with our local cookies. cookies = self.cookies or dict() + print ' %s' % str(cookies) # Add new cookies from the server. if 'set-cookie' in response.headers: diff --git a/requests/sessions.py b/requests/sessions.py index 9a10808e..9a68e618 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -9,12 +9,10 @@ requests (cookies, auth, proxies). """ -import cookielib - from .defaults import defaults from .models import Request from .hooks import dispatch_hook -from .utils import add_dict_to_cookiejar, cookiejar_from_dict, header_expand +from .utils import header_expand from .packages.urllib3.poolmanager import PoolManager @@ -87,7 +85,7 @@ class Session(object): ) # Set up a CookieJar to be used by default - self.cookies = cookielib.FileCookieJar() + self.cookies = {} def __repr__(self): return '' % (id(self)) @@ -135,11 +133,6 @@ class Session(object): if cookies is None: cookies = {} - if isinstance(cookies, dict): - cookies = add_dict_to_cookiejar(self.cookies, cookies) - - cookies = cookiejar_from_dict(cookies) - # Expand header values if headers: for k, v in headers.items() or {}: diff --git a/test_requests.py b/test_requests.py index 261a01c0..266c20a2 100755 --- a/test_requests.py +++ b/test_requests.py @@ -4,13 +4,12 @@ from __future__ import with_statement import time -import cookielib import os import unittest import requests import envoy -from urllib2 import HTTPError +from requests import HTTPError try: import omnijson as json @@ -235,7 +234,8 @@ class RequestsTestSuite(unittest.TestCase): for service in SERVICES: r = requests.get(service('status', '404')) - print r.status_code + # print r.status_code + # r.raise_for_status() self.assertEqual(r.ok, False) @@ -510,6 +510,15 @@ class RequestsTestSuite(unittest.TestCase): assert params3['b'] in r3.content assert params3['c'] in r3.content + def test_cookies(self): + + s = requests.session() + r = s.get(httpbin('cookies', 'set', 'face', 'book')) + print r.headers + print r.history[0].cookies + print r.content + print r.url + if __name__ == '__main__': unittest.main() From 6fefffd03ed2a8ebfa882fff48fda41f29251d1f Mon Sep 17 00:00:00 2001 From: Juan Riaza Date: Sun, 6 Nov 2011 22:02:14 +0100 Subject: [PATCH 170/255] #GH-241 --- requests/async.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/requests/async.py b/requests/async.py index 92839c30..16908e27 100644 --- a/requests/async.py +++ b/requests/async.py @@ -12,6 +12,7 @@ by gevent. All API methods return a ``Request`` instance (as opposed to try: import gevent from gevent import monkey as curious_george + from gevent.pool import Pool except ImportError: raise RuntimeError('Gevent is required for requests.async.') @@ -61,15 +62,21 @@ delete = patched(api.delete) request = patched(api.request) -def map(requests, prefetch=True): +def map(requests, prefetch=True, size=None): """Concurrently converts a list of Requests to Responses. :param requests: a collection of Request objects. :param prefetch: If False, the content will not be downloaded immediately. + :param size: Specifies the number of requests to make at a time. If None, no throttling occurs. """ - jobs = [gevent.spawn(send, r) for r in requests] - gevent.joinall(jobs) + if size: + pool = Pool(size) + pool.map(send, requests) + pool.join() + else: + jobs = [gevent.spawn(send, r) for r in requests] + gevent.joinall(jobs) if prefetch: [r.response.content for r in requests] From 73ba48be2e51e2dea2f4e6c924495796c112a9a3 Mon Sep 17 00:00:00 2001 From: jbrendel Date: Tue, 8 Nov 2011 12:31:18 +1300 Subject: [PATCH 171/255] Added support for OPTIONS method. --- AUTHORS | 3 ++- requests/__init__.py | 2 +- requests/api.py | 13 ++++++++++++- requests/async.py | 3 ++- requests/sessions.py | 13 ++++++++++++- 5 files changed, 29 insertions(+), 5 deletions(-) diff --git a/AUTHORS b/AUTHORS index 131c0f61..2ebcdfda 100644 --- a/AUTHORS +++ b/AUTHORS @@ -49,4 +49,5 @@ Patches and Suggestions - Daniel Hengeveld - Dan Head - Bruno Renié -- David Fischer \ No newline at end of file +- David Fischer +- Juergen Brendel diff --git a/requests/__init__.py b/requests/__init__.py index 1140a523..d496b98d 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -24,7 +24,7 @@ __copyright__ = 'Copyright 2011 Kenneth Reitz' from . import utils from .models import Request, Response -from .api import request, get, head, post, patch, put, delete +from .api import request, get, options, head, post, patch, put, delete from .sessions import session from .status_codes import codes from .exceptions import ( diff --git a/requests/api.py b/requests/api.py index a139f3d5..764e4906 100644 --- a/requests/api.py +++ b/requests/api.py @@ -13,7 +13,7 @@ This module implements the Requests API. from .sessions import session -__all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete') +__all__ = ('request', 'get', 'options', 'head', 'post', 'patch', 'put', 'delete') def request(method, url, @@ -67,6 +67,17 @@ def get(url, **kwargs): return request('GET', url, **kwargs) +def options(url, **kwargs): + """Sends a OPTIONS request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param **kwargs: Optional arguments that ``request`` takes. + """ + + kwargs.setdefault('allow_redirects', True) + return request('OPTIONS', url, **kwargs) + + def head(url, **kwargs): """Sends a HEAD request. Returns :class:`Response` object. diff --git a/requests/async.py b/requests/async.py index 92839c30..cef383f6 100644 --- a/requests/async.py +++ b/requests/async.py @@ -24,7 +24,7 @@ from .hooks import dispatch_hook __all__ = ( 'map', - 'get', 'head', 'post', 'put', 'patch', 'delete', 'request' + 'get', 'options', 'head', 'post', 'put', 'patch', 'delete', 'request' ) @@ -53,6 +53,7 @@ def send(r, pools=None): # Patched requests.api functions. get = patched(api.get) +options = patched(api.options) head = patched(api.head) post = patched(api.post) put = patched(api.put) diff --git a/requests/sessions.py b/requests/sessions.py index eaeb96e4..7cc8479c 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -188,6 +188,17 @@ class Session(object): return self.request('GET', url, **kwargs) + def options(self, url, **kwargs): + """Sends a OPTIONS request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param **kwargs: Optional arguments that ``request`` takes. + """ + + kwargs.setdefault('allow_redirects', True) + return self.request('OPTIONS', url, **kwargs) + + def head(self, url, **kwargs): """Sends a HEAD request. Returns :class:`Response` object. @@ -246,4 +257,4 @@ class Session(object): def session(**kwargs): """Returns a :class:`Session` for context-management.""" - return Session(**kwargs) \ No newline at end of file + return Session(**kwargs) From 8de59f03a09be6567ee7a877ce9bbbabbe0b9a50 Mon Sep 17 00:00:00 2001 From: Joseph McCullough Date: Tue, 8 Nov 2011 01:37:39 -0600 Subject: [PATCH 172/255] Added POST to quick start. Modified GET in doc to use httpbin --- docs/user/quickstart.rst | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 17e503a2..b730c6ed 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -25,19 +25,40 @@ Making a standard request with Requests is very simple. Let's get GitHub's public timeline :: - r = requests.get('https://github.com/timeline.json') + r = requests.get("http://httpbin.org/get") Now, we have a :class:`Response` object called ``r``. We can get all the information we need from this. + Response Content ---------------- We can read the content of the server's response:: >>> r.content - '[{"repository":{"open_issues":0,"url":"https://github.com/... + '{\n "url": "http://httpbin.org/get", \n "headers": ... + + +Make a POST Request +------------------ + +POST requests are equally simple :: + + >>> r = requests.post("http://httpbin.org/post") + + +Suppose you want to send data over HTTP. Simply pass a data +argument to the requests.post method with your dictionary :: + + >>> dataDict = {"key1":"value1", "key2":"value2"} + >>> r = requests.post("http://httpbin.org/post", data=dataDict) + >>> r.content + '{\n "origin": "::ffff:YourIpAddress", \n "files": {}, \n "form": {\n "key2": "value2", \n "key1": "value1"\n }, + +Note the data= argument is equivalent to -d in cURL scripts. dataDict will +be form-encoded. Response Status Codes @@ -165,3 +186,4 @@ Requests supports it!:: ----------------------- Ready for more? Check out the :ref:`advanced ` section. + From 827b2340a242c0882df2173744a8951f7ca88b14 Mon Sep 17 00:00:00 2001 From: Joseph McCullough Date: Tue, 8 Nov 2011 02:23:32 -0600 Subject: [PATCH 173/255] Forgot to delete line mentioning Github in GET section --- docs/user/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index b730c6ed..4bebe95c 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -23,7 +23,7 @@ Make a GET Request Making a standard request with Requests is very simple. -Let's get GitHub's public timeline :: +Let's use httpbin to test our requests. :: r = requests.get("http://httpbin.org/get") From 1031ec494212076a5e1b9d5398a95c96cb7c561e Mon Sep 17 00:00:00 2001 From: Joseph McCullough Date: Tue, 8 Nov 2011 12:10:06 -0600 Subject: [PATCH 174/255] Reverted GET example back to Github. --- docs/user/quickstart.rst | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 4bebe95c..69e86c40 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -23,22 +23,21 @@ Make a GET Request Making a standard request with Requests is very simple. -Let's use httpbin to test our requests. :: +Let's get GitHub's public timeline :: - r = requests.get("http://httpbin.org/get") + r = requests.get('https://github.com/timeline.json') Now, we have a :class:`Response` object called ``r``. We can get all the information we need from this. - Response Content ---------------- We can read the content of the server's response:: >>> r.content - '{\n "url": "http://httpbin.org/get", \n "headers": ... + '[{"repository":{"open_issues":0,"url":"https://github.com/... Make a POST Request @@ -55,12 +54,11 @@ argument to the requests.post method with your dictionary :: >>> dataDict = {"key1":"value1", "key2":"value2"} >>> r = requests.post("http://httpbin.org/post", data=dataDict) >>> r.content - '{\n "origin": "::ffff:YourIpAddress", \n "files": {}, \n "form": {\n "key2": "value2", \n "key1": "value1"\n }, + '{\n "origin": "::ffff:YourIpAddress", \n "files": {}, \n "form": {\n "key2": "value2", \n "key1": "value1"\n }, ... Note the data= argument is equivalent to -d in cURL scripts. dataDict will be form-encoded. - Response Status Codes --------------------- @@ -186,4 +184,3 @@ Requests supports it!:: ----------------------- Ready for more? Check out the :ref:`advanced ` section. - From f5bbb9714d66103280d0ca10742877f4522787ff Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 10:24:41 -0500 Subject: [PATCH 175/255] Note about terrible DNS providers for testing --- test_requests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test_requests.py b/test_requests.py index ebff33dd..d5c9eaa1 100755 --- a/test_requests.py +++ b/test_requests.py @@ -531,7 +531,10 @@ class RequestsTestSuite(unittest.TestCase): def test_invalid_content(self): - r = requests.get('http://somedomainthatclearlydoesntexistg.com') + # WARNING: if you're using a terrible DNS provider (comcast), + # this will fail. + r = requests.get('http://somedomainthatclearlydoesntexistg.com', allow_redirects=False) + assert r.content == None From 498b446f141675026ae7364b922f03cde1de728c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 14:57:00 -0800 Subject: [PATCH 176/255] Models cleanup --- requests/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index bd3d9abc..e5357c6a 100644 --- a/requests/models.py +++ b/requests/models.py @@ -8,7 +8,6 @@ requests.models """ import urllib -import socket import zlib from Cookie import SimpleCookie @@ -183,7 +182,6 @@ class Request(object): # Start off with our local cookies. cookies = self.cookies or dict() - print ' %s' % str(cookies) # Add new cookies from the server. if 'set-cookie' in response.headers: From 358478eb8881e3ebdef6afae09ee8ba934f66ee9 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 14:57:38 -0800 Subject: [PATCH 177/255] Cookie persistence! --- requests/sessions.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/requests/sessions.py b/requests/sessions.py index 9a68e618..77a26e5b 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -87,6 +87,9 @@ class Session(object): # Set up a CookieJar to be used by default self.cookies = {} + if cookies is not None: + self.cookies.update(cookies) + def __repr__(self): return '' % (id(self)) @@ -165,6 +168,7 @@ class Session(object): # Arguments manipulation hook. args = dispatch_hook('args', args['hooks'], args) + # Create the (empty) response. r = Request(**args) # Don't send if asked nicely. @@ -174,6 +178,10 @@ class Session(object): # Send the HTTP Request. r.send() + # Send any cookies back up the to the session. + self.cookies.update(r.response.cookies) + + # Return the response. return r.response From e6490039749ae6b1a3305a9bab2b826979b5fb56 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 15:02:53 -0800 Subject: [PATCH 178/255] test the cookies! --- test_requests.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/test_requests.py b/test_requests.py index 266c20a2..a3e64ec7 100755 --- a/test_requests.py +++ b/test_requests.py @@ -59,7 +59,6 @@ class RequestsTestSuite(unittest.TestCase): # self.httpbin.kill() def test_entry_points(self): - import requests requests.session requests.session().get @@ -482,6 +481,30 @@ class RequestsTestSuite(unittest.TestCase): self.assertEqual(r2.status_code, 200) + def test_session_persistent_cookies(self): + + s = requests.session() + + # Internally dispatched cookies are sent. + _c = {'kenneth': 'reitz', 'bessie': 'monke'} + r = s.get(httpbin('cookies'), cookies=_c) + r = s.get(httpbin('cookies')) + + # Those cookies persist transparently. + c = json.loads(r.content).get('cookies') + assert c == _c + + # Double check. + r = s.get(httpbin('cookies'), cookies={}) + c = json.loads(r.content).get('cookies') + assert c == _c + + # Remove a cookie by setting it's value to None. + r = s.get(httpbin('cookies'), cookies={'bessie': None}) + c = json.loads(r.content).get('cookies') + del _c['bessie'] + assert c == _c + def test_session_persistent_params(self): From f9141f51ec937da874aa6799393acb78a0247675 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 15:04:13 -0800 Subject: [PATCH 179/255] Make sure that session-level cookies work. --- test_requests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test_requests.py b/test_requests.py index a3e64ec7..4e7b5b32 100755 --- a/test_requests.py +++ b/test_requests.py @@ -506,6 +506,12 @@ class RequestsTestSuite(unittest.TestCase): assert c == _c + s = requests.session(cookies=_c) + c = json.loads(r.content).get('cookies') + assert c == _c + + + def test_session_persistent_params(self): params = {'a': 'a_test'} From 6a51f6b2f8af5b3e68a9b3f2edfd4c673dadc3b1 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 15:30:33 -0800 Subject: [PATCH 180/255] add 'max_retries' configuration --- requests/defaults.py | 1 + requests/models.py | 70 ++++++++++++++------------------------------ 2 files changed, 23 insertions(+), 48 deletions(-) diff --git a/requests/defaults.py b/requests/defaults.py index 90faabee..0d53dfc9 100644 --- a/requests/defaults.py +++ b/requests/defaults.py @@ -38,3 +38,4 @@ defaults['timeout_fallback'] = True # defaults['keep_alive'] = True defaults['pool_connections'] = 10 defaults['pool_maxsize'] = 1 +defaults['max_retries'] = 0 diff --git a/requests/models.py b/requests/models.py index 5ec06a66..275486a8 100644 --- a/requests/models.py +++ b/requests/models.py @@ -14,12 +14,14 @@ from Cookie import SimpleCookie from urlparse import urlparse, urlunparse, urljoin from datetime import datetime + from .auth import dispatch as auth_dispatch from .hooks import dispatch_hook from .structures import CaseInsensitiveDict from .status_codes import codes -from .exceptions import Timeout, URLRequired, TooManyRedirects, HTTPError - +from .packages.urllib3.exceptions import MaxRetryError +from .exceptions import ( + Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError) from .utils import ( dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip) @@ -129,39 +131,6 @@ class Request(object): return '' % (self.method) - # def _get_opener(self): - # """Creates appropriate opener object for urllib2.""" - - # _handlers = [] - - # if self.cookies is not None: - # _handlers.append(urllib2.HTTPCookieProcessor(self.cookies)) - - # if self.proxies: - # _handlers.append(urllib2.ProxyHandler(self.proxies)) - - # _handlers.append(HTTPRedirectHandler) - - # if not _handlers: - # return urllib2.urlopen - - # if self.data or self.files: - # _handlers.extend(get_handlers()) - - # opener = urllib2.build_opener(*_handlers) - - # if self.headers: - # # Allow default headers in the opener to be overloaded - # normal_keys = [k.capitalize() for k in self.headers] - # for key, val in opener.addheaders[:]: - # if key not in normal_keys: - # continue - # # Remove it, we have a value to take its place - # opener.addheaders.remove((key, val)) - - # return opener.open - - def _build_response(self, resp, is_error=False): """Build internal :class:`Response ` object from given response. @@ -219,8 +188,6 @@ class Request(object): ((r.status_code is codes.see_other) or (self.allow_redirects)) ): - # r.raw.close() - if not len(history) < self.config.get('max_redirects'): raise TooManyRedirects() @@ -402,17 +369,24 @@ class Request(object): # Attach Cookie header to request. self.headers['Cookie'] = cookie_header - # Create the connection. - r = conn.urlopen( - method=self.method, - url=url, - body=body, - headers=self.headers, - redirect=False, - assert_same_host=False, - preload_content=False, - decode_content=False - ) + try: + # Create the connection. + r = conn.urlopen( + method=self.method, + url=url, + body=body, + headers=self.headers, + redirect=False, + assert_same_host=False, + preload_content=False, + decode_content=False, + retries=self.config.get('max_retries', 0) + ) + except MaxRetryError, e: + if self.config.get('safe_mode', False): + pass + raise ConnectionError(e) + self._build_response(r) From 1d2abea94b2323352e18648154a163c96136b689 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 15:30:47 -0800 Subject: [PATCH 181/255] new ConnectionError for bad APIs :) --- requests/exceptions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/exceptions.py b/requests/exceptions.py index 16acd925..d20a95cd 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -15,8 +15,8 @@ class RequestException(Exception): class HTTPError(RequestException): """An HTTP error occured.""" -class AuthenticationError(RequestException): - """The authentication credentials provided were invalid.""" +class ConnectionError(RequestException): + """A Connection error occured.""" class Timeout(RequestException): """The request timed out.""" From 5b73f2ea6100277f6400f399f585117b47068b4a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 15:43:54 -0800 Subject: [PATCH 182/255] infos --- requests/sessions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/requests/sessions.py b/requests/sessions.py index 77a26e5b..ddac7860 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -87,6 +87,7 @@ class Session(object): # Set up a CookieJar to be used by default self.cookies = {} + # Add passed cookies in. if cookies is not None: self.cookies.update(cookies) From 0f2dcff72573c80fbabb022ba2f1421901269f3a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 15:44:14 -0800 Subject: [PATCH 183/255] add query data yourself, foo' --- test_requests.py | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/test_requests.py b/test_requests.py index 7b10a790..300779e4 100755 --- a/test_requests.py +++ b/test_requests.py @@ -323,21 +323,6 @@ class RequestsTestSuite(unittest.TestCase): self.assertEquals(rbody.get('data'), '') - def test_nonurlencoded_post_querystring(self): - - for service in SERVICES: - - r = requests.post(service('post'), params='fooaowpeuf') - - self.assertEquals(r.status_code, 200) - self.assertEquals(r.headers['content-type'], 'application/json') - self.assertEquals(r.url, service('post?fooaowpeuf')) - - rbody = json.loads(r.content) - self.assertEquals(rbody.get('form'), {}) # No form supplied - self.assertEquals(rbody.get('data'), '') - - def test_urlencoded_post_query_and_data(self): for service in SERVICES: @@ -356,16 +341,14 @@ class RequestsTestSuite(unittest.TestCase): self.assertEquals(rbody.get('data'), '') - def test_nonurlencoded_post_query_and_data(self): + def test_nonurlencoded_postdata(self): for service in SERVICES: - r = requests.post(service('post'), - params='fooaowpeuf', data="foobar") + r = requests.post(service('post'), data="foobar") self.assertEquals(r.status_code, 200) self.assertEquals(r.headers['content-type'], 'application/json') - self.assertEquals(r.url, service('post?fooaowpeuf')) rbody = json.loads(r.content) @@ -539,24 +522,37 @@ class RequestsTestSuite(unittest.TestCase): assert params3['b'] in r3.content assert params3['c'] in r3.content + def test_cookies(self): s = requests.session() r = s.get(httpbin('cookies', 'set', 'face', 'book')) - print r.headers - print r.history[0].cookies - print r.content - print r.url + # print r.headers + # print r.history[0].cookies + # print r.content + # print r.url + def test_invalid_content(self): # WARNING: if you're using a terrible DNS provider (comcast), # this will fail. - r = requests.get('http://somedomainthatclearlydoesntexistg.com', allow_redirects=False) + try: + hah = 'http://somedomainthatclearlydoesntexistg.com' + r = requests.get(hah, allow_redirects=False) + except requests.ConnectionError: + pass # \o/ + else: + assert False + + config = {'safe_mode': True} + r = requests.get(hah, allow_redirects=False, config=config) assert r.content == None + + if __name__ == '__main__': unittest.main() From 35ed9b5f38ea9fa210658d555dbb8e1a71fb66d8 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 15:44:55 -0800 Subject: [PATCH 184/255] dict(o or []) --- requests/models.py | 48 ++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/requests/models.py b/requests/models.py index 275486a8..c38f3799 100644 --- a/requests/models.py +++ b/requests/models.py @@ -61,7 +61,7 @@ class Request(object): self.url = url #: Dictionary of HTTP Headers to attach to the :class:`Request `. - self.headers = headers + self.headers = dict(headers or []) #: Dictionary of files to multipart upload (``{filename: content}``). self.files = files @@ -85,7 +85,7 @@ class Request(object): self.allow_redirects = allow_redirects # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) - self.proxies = proxies + self.proxies = dict(proxies or []) self.data, self._enc_data = self._encode_params(data) self.params, self._enc_params = self._encode_params(params) @@ -99,10 +99,10 @@ class Request(object): self.auth = auth_dispatch(auth) #: CookieJar to attach to :class:`Request `. - self.cookies = cookies + self.cookies = dict(cookies or []) #: Dictionary of configurations for this request. - self.config = config + self.config = dict(config or []) #: True if Request has been sent. self.sent = False @@ -144,27 +144,29 @@ class Request(object): # Pass settings over. response.config = self.config - # Fallback to None if there's no staus_code, for whatever reason. - response.status_code = getattr(resp, 'status', None) + if resp: - # Make headers case-insensitive. - response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) + # Fallback to None if there's no staus_code, for whatever reason. + response.status_code = getattr(resp, 'status', None) - # Start off with our local cookies. - cookies = self.cookies or dict() + # Make headers case-insensitive. + response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) - # Add new cookies from the server. - if 'set-cookie' in response.headers: - cookie_header = response.headers['set-cookie'] + # Start off with our local cookies. + cookies = self.cookies or dict() - c = SimpleCookie() - c.load(cookie_header) + # Add new cookies from the server. + if 'set-cookie' in response.headers: + cookie_header = response.headers['set-cookie'] - for k,v in c.items(): - cookies.update({k: v.value}) + c = SimpleCookie() + c.load(cookie_header) - # Save cookies in Response. - response.cookies = cookies + for k,v in c.items(): + cookies.update({k: v.value}) + + # Save cookies in Response. + response.cookies = cookies # Save original resopnse for later. response.raw = resp @@ -383,10 +385,10 @@ class Request(object): retries=self.config.get('max_retries', 0) ) except MaxRetryError, e: - if self.config.get('safe_mode', False): - pass - raise ConnectionError(e) - + if not self.config.get('safe_mode', False): + raise ConnectionError(e) + else: + r = None self._build_response(r) From 99f0ef47668b7a8a62f1b64d064c89a149cf105f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 15:46:01 -0800 Subject: [PATCH 185/255] add safe_mode for @bitprophet :) --- requests/defaults.py | 1 + 1 file changed, 1 insertion(+) diff --git a/requests/defaults.py b/requests/defaults.py index 0d53dfc9..c99786f0 100644 --- a/requests/defaults.py +++ b/requests/defaults.py @@ -39,3 +39,4 @@ defaults['timeout_fallback'] = True defaults['pool_connections'] = 10 defaults['pool_maxsize'] = 1 defaults['max_retries'] = 0 +defaults['safe_mode'] = False From 147c24154e4315750f7513ca387f4b843bcc85d2 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 15:46:45 -0800 Subject: [PATCH 186/255] better defaults for "models" --- requests/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index c38f3799..3a30aa03 100644 --- a/requests/models.py +++ b/requests/models.py @@ -76,6 +76,7 @@ class Request(object): #: Dictionary or byte of querystring data to attach to the #: :class:`Request `. self.params = None + self.params = dict(params or []) #: True if :class:`Request ` is part of a redirect chain (disables history #: and HTTPError storage). @@ -440,7 +441,7 @@ class Response(object): self.request = None #: A dictionary of Cookies the server sent back. - self.cookies = None + self.cookies = {} #: Dictionary of configurations for this request. self.config = None From 7192b8022e6273c60de2b94d6e03015eaefd5c75 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 15:46:55 -0800 Subject: [PATCH 187/255] import ConnectionError --- requests/__init__.py | 4 ++-- requests/models.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requests/__init__.py b/requests/__init__.py index 05c9b82c..eb6639f9 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -28,6 +28,6 @@ from .api import request, get, head, post, patch, put, delete from .sessions import session from .status_codes import codes from .exceptions import ( - RequestException, AuthenticationError, Timeout, URLRequired, - TooManyRedirects, HTTPError + RequestException, Timeout, URLRequired, + TooManyRedirects, HTTPError, ConnectionError ) diff --git a/requests/models.py b/requests/models.py index 3a30aa03..3b711cf5 100644 --- a/requests/models.py +++ b/requests/models.py @@ -444,7 +444,7 @@ class Response(object): self.cookies = {} #: Dictionary of configurations for this request. - self.config = None + self.config = {} def __repr__(self): From 0c4b973dcf74a3c2fe0c3123e7d4637d8ccea481 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 15:53:45 -0800 Subject: [PATCH 188/255] keep_alive is now disable-able! --- requests/defaults.py | 2 +- requests/models.py | 10 ++++++++-- test_requests.py | 5 ++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/requests/defaults.py b/requests/defaults.py index c99786f0..49db114c 100644 --- a/requests/defaults.py +++ b/requests/defaults.py @@ -35,7 +35,7 @@ defaults['timeout'] = None defaults['max_redirects'] = 30 defaults['decode_unicode'] = True defaults['timeout_fallback'] = True -# defaults['keep_alive'] = True +defaults['keep_alive'] = True defaults['pool_connections'] = 10 defaults['pool_maxsize'] = 1 defaults['max_retries'] = 0 diff --git a/requests/models.py b/requests/models.py index 3b711cf5..1c4d5758 100644 --- a/requests/models.py +++ b/requests/models.py @@ -20,6 +20,7 @@ from .hooks import dispatch_hook from .structures import CaseInsensitiveDict from .status_codes import codes from .packages.urllib3.exceptions import MaxRetryError +from .packages.urllib3 import connectionpool from .exceptions import ( Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError) from .utils import ( @@ -352,7 +353,11 @@ class Request(object): self.__dict__.update(r.__dict__) - conn = self._poolmanager.connection_from_url(url) + if self.config.get('keep_alive'): + conn = self._poolmanager.connection_from_url(url) + else: + conn = connectionpool.connection_from_url(url) + print 'NO CONNECTION FOR YOU1' if not self.sent or anyway: @@ -373,7 +378,7 @@ class Request(object): self.headers['Cookie'] = cookie_header try: - # Create the connection. + # Send the request. r = conn.urlopen( method=self.method, url=url, @@ -385,6 +390,7 @@ class Request(object): decode_content=False, retries=self.config.get('max_retries', 0) ) + except MaxRetryError, e: if not self.config.get('safe_mode', False): raise ConnectionError(e) diff --git a/test_requests.py b/test_requests.py index 300779e4..8b0f7c7a 100755 --- a/test_requests.py +++ b/test_requests.py @@ -57,6 +57,8 @@ class RequestsTestSuite(unittest.TestCase): def tearDown(self): """Teardown.""" # self.httpbin.kill() + pass + def test_entry_points(self): @@ -551,8 +553,5 @@ class RequestsTestSuite(unittest.TestCase): assert r.content == None - - - if __name__ == '__main__': unittest.main() From 52a95cb93ab8b6c2c32398bd440fd9434232a13a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 16:24:48 -0800 Subject: [PATCH 189/255] header and cookie fixes for redirects --- requests/models.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/requests/models.py b/requests/models.py index 1c4d5758..734e0b8e 100644 --- a/requests/models.py +++ b/requests/models.py @@ -184,6 +184,8 @@ class Request(object): history = [] r = build(resp) + cookies = self.cookies + self.cookies.update(r.cookies) if r.status_code in REDIRECT_STATI and not self.redirect: @@ -215,26 +217,40 @@ class Request(object): else: method = self.method + # Remove the cookie headers that were sent. + headers = self.headers + try: + del headers['Cookie'] + except KeyError: + pass + request = Request( url=url, - headers=self.headers, + headers=headers, files=self.files, method=method, - # data=self.data, # params=self.params, auth=self._auth, - cookies=self.cookies, + cookies=cookies, redirect=True, config=self.config, _poolmanager=self._poolmanager ) + request.send() + cookies.update(request.response.cookies) r = request.response + self.cookies.update(r.cookies) r.history = history self.response = r self.response.request = self + # print locals() + self.response.cookies.update(self.cookies) + # print cookies + # print self.response.cookies + # print '!!!' @staticmethod @@ -348,16 +364,17 @@ class Request(object): if self.auth: auth_func, auth_args = self.auth + # Allow auth to make its changes. r = auth_func(self, *auth_args) + # Update self to reflect the auth changes. self.__dict__.update(r.__dict__) - + # Check to see if keep_alive is allowed. if self.config.get('keep_alive'): conn = self._poolmanager.connection_from_url(url) else: conn = connectionpool.connection_from_url(url) - print 'NO CONNECTION FOR YOU1' if not self.sent or anyway: From 7a1a40e4455c3ee07bc0860b5e11a43ee2ec7c6e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 16:25:20 -0800 Subject: [PATCH 190/255] ultra cookie compatibility --- requests/models.py | 4 ---- test_requests.py | 26 ++++++++++++++------------ 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/requests/models.py b/requests/models.py index 734e0b8e..c04e4c51 100644 --- a/requests/models.py +++ b/requests/models.py @@ -246,11 +246,7 @@ class Request(object): self.response = r self.response.request = self - # print locals() self.response.cookies.update(self.cookies) - # print cookies - # print self.response.cookies - # print '!!!' @staticmethod diff --git a/test_requests.py b/test_requests.py index 8b0f7c7a..40fa82e5 100755 --- a/test_requests.py +++ b/test_requests.py @@ -490,11 +490,24 @@ class RequestsTestSuite(unittest.TestCase): del _c['bessie'] assert c == _c - + # Test session-level cookies. s = requests.session(cookies=_c) + r = s.get(httpbin('cookies')) c = json.loads(r.content).get('cookies') assert c == _c + # Have the server set a cookie. + r = s.get(httpbin('cookies', 'set', 'k', 'v'), allow_redirects=True) + c = json.loads(r.content).get('cookies') + + assert 'k' in c + + # And server-set cookie persistience. + r = s.get(httpbin('cookies')) + c = json.loads(r.content).get('cookies') + + assert 'k' in c + def test_session_persistent_params(self): @@ -524,17 +537,6 @@ class RequestsTestSuite(unittest.TestCase): assert params3['b'] in r3.content assert params3['c'] in r3.content - - def test_cookies(self): - - s = requests.session() - r = s.get(httpbin('cookies', 'set', 'face', 'book')) - # print r.headers - # print r.history[0].cookies - # print r.content - # print r.url - - def test_invalid_content(self): # WARNING: if you're using a terrible DNS provider (comcast), From 2131ba4dda0197b5795096b703fee3d0980b9c42 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 16:26:36 -0800 Subject: [PATCH 191/255] check! --- requests/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index c04e4c51..d871e719 100644 --- a/requests/models.py +++ b/requests/models.py @@ -350,8 +350,6 @@ class Request(object): body = self._enc_data content_type = 'application/x-www-form-urlencoded' - # TODO: Setup cookies. - # Add content-type if it wasn't explicitly provided. if (content_type) and (not 'content-type' in self.headers): self.headers['Content-Type'] = content_type From 92a8db6da0fc91b09fd7ff4068dad951aa0a0c8c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 16:52:10 -0800 Subject: [PATCH 192/255] httplib + urllib3 --- docs/index.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 73da870c..df7ab830 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -33,15 +33,14 @@ See `the same code, without Requests `_. Requests allow you to send **HEAD**, **GET**, **POST**, **PUT**, **PATCH**, and **DELETE** HTTP requests. You can add headers, form data, multipart files, and parameters with simple Python dictionaries, and access the -response data in the same way. It's powered by :py:class:`urllib2`, but it does -all the hard work and crazy hacks for you. +response data in the same way. It's powered by :py:class:`httplib` and urllib3, and it works just as you'd expect. Testimonials ------------ `The Washington Post `_, `Twitter, Inc `_, a U.S. Federal Institution, -NIH, +NIH, `Readability `_, and `Work for Pie `_ use Requests internally. From 4e8b68086559703e2266475cbdaee847039a3a1b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 16:52:20 -0800 Subject: [PATCH 193/255] explain sessions and keep alive --- docs/user/advanced.rst | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 90d11433..42f60a9c 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -37,10 +37,35 @@ Sessions can also be used to provide default data to the request methods:: c.get('http://httpbin.org/headers', headers={'x-test2': 'true'}) -.. admonition:: Global Settings +Any dictionaries that you pass to a request method will be merged with the session-level values that are set. The method-level parameters override session parameters. - Certain parameters are best set in the ``config`` dictionary - (e.g. user agent header). +.. admonition:: Remove a Value From a Dict Parameter + + Sometimes you'll want to omit session-level keys from a dict parameter. To do this, you simply set that key's value to ``None`` in the method-level parameter. It will automatically be omitted. + +All values that are contained within a session are directly available to you: + + Session attibutes: + ``auth``, ``config``, ``cookies``, ``headers``, ``hooks``, ``keep_alive``, ``params``, ``proxies``, ''timeout`` + + + +Configuring Requests +-------------------- + +Sometimes you may want to configure a request to customize it's behavior. To do +this, you can pass in a ``config`` dictionary to a request or session. + + +Keep-Alive +---------- + +Excellent news — keep alive is 100% automatic within a session! Couldn't be easier. + +If you'd like to disable keep-alive, you can simply set the ``keep_alive`` configuration to ``False``:: + + s = requests.session() + s.config['keep_alive'] = False Asynchronous Requests From e725f30625563e32a1a8f31438d2820baaa83c28 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 16:52:27 -0800 Subject: [PATCH 194/255] document defaults --- requests/defaults.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/requests/defaults.py b/requests/defaults.py index 49db114c..3840097c 100644 --- a/requests/defaults.py +++ b/requests/defaults.py @@ -29,14 +29,31 @@ defaults['base_headers'] = { 'Accept': '*/*' } -defaults['proxies'] = {} + +#: Stream to log requests to. defaults['verbose'] = None + +#: Seconds until timeout. defaults['timeout'] = None + +#: Maximum number of redirects allowed within a request. defaults['max_redirects'] = 30 + +#: Should Requests decode unicode? defaults['decode_unicode'] = True -defaults['timeout_fallback'] = True + +#: Reuse HTTP Connections? defaults['keep_alive'] = True + +#: The number of active HTTP connection pools to use at a time. defaults['pool_connections'] = 10 + +#: The maximium size of an HTTP connection pool. defaults['pool_maxsize'] = 1 + +#: The number of times a request should be retried in the event of a +#: connection failure. defaults['max_retries'] = 0 + +#: If true, Requests will catch all errors. defaults['safe_mode'] = False From 53bf544050c790c43166fe8ff02d10c6c8983323 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 17:15:44 -0800 Subject: [PATCH 195/255] better api docs --- docs/api.rst | 71 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 9b1b10f2..adbd53f7 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -17,6 +17,15 @@ All of Request's functionality can be accessed by these 7 methods. They all return an instance of the :class:`Response ` object. .. autofunction:: request + +--------------------- + + +.. autoclass:: Response + :inherited-members: + +--------------------- + .. autofunction:: head .. autofunction:: get .. autofunction:: post @@ -25,11 +34,31 @@ They all return an instance of the :class:`Response ` object. .. autofunction:: delete ------------ +----------------- + +.. autofunction:: session + + + +Exceptions +~~~~~~~~~~ + +.. module:: requests + +.. autoexception:: RequestException +.. autoexception:: ConnectionError +.. autoexception:: HTTPError +.. autoexception:: URLRequired +.. autoexception:: TooManyRedirects + + + +Configurations +-------------- + +.. automodule:: requests.defaults -.. autoclass:: Response - :inherited-members: Async ----- @@ -56,6 +85,22 @@ Requests. .. module:: requests.utils +Status Code Lookup +~~~~~~~~~~~~~~~~~~ + +.. autofunction:: requests.codes + +:: + + >>> requests.codes['temporary_redirect'] + 301 + + >>> requests.codes.teapot + 416 + + >>> requests.codes['\o/'] + 416 + Cookies ~~~~~~~ @@ -80,26 +125,16 @@ These items are an internal component to Requests, and should never be seen by the end user (developer). This part of the API documentation exists for those who are extending the functionality of Requests. -Exceptions -~~~~~~~~~~ - -.. module:: requests - -.. autoexception:: HTTPError - -.. autoexception:: RequestException - -.. autoexception:: AuthenticationError -.. autoexception:: URLRequired -.. autoexception:: InvalidMethod -.. autoexception:: TooManyRedirects - - Classes ~~~~~~~ +.. autoclass:: requests.Response + :inherited-members: + .. autoclass:: requests.Request :inherited-members: +.. autoclass:: requests.Session + :inherited-members: From 17088763e7ec86fc8e378e23a386199281c31e47 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 17:15:54 -0800 Subject: [PATCH 196/255] better advanced quick start --- docs/user/advanced.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 42f60a9c..a6019ee9 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -46,7 +46,7 @@ Any dictionaries that you pass to a request method will be merged with the sessi All values that are contained within a session are directly available to you: Session attibutes: - ``auth``, ``config``, ``cookies``, ``headers``, ``hooks``, ``keep_alive``, ``params``, ``proxies``, ''timeout`` + ``auth``, ``config``, ``cookies``, ``headers``, ``hooks``, ``keep_alive``, ``params``, ``proxies``, ``timeout`` @@ -71,7 +71,7 @@ If you'd like to disable keep-alive, you can simply set the ``keep_alive`` confi Asynchronous Requests ---------------------- -Requests has first-class support for non-blocking i/o requests, powered +Requests has first-class support for concurrent requests, powered by gevent. This allows you to send a bunch of HTTP requests at the same First, let's import the async module. Heads up — if you don't have @@ -102,6 +102,12 @@ will also guarantee execution of the ``response`` hook, described below. :: >>> async.map(rs) [, , , ] +.. admonition:: Throttling + + The ``map`` function also takes a ``size`` parameter, that specifies the nubmer of connections to make at a time:: + + async.map(rs, size=5) + Event Hooks ----------- From ce42b338f4a1ddcb52c7b7624a43163ebe9d527a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 17:16:03 -0800 Subject: [PATCH 197/255] Configurations --- requests/defaults.py | 38 +++++++++++--------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/requests/defaults.py b/requests/defaults.py index 3840097c..e3f3ba37 100644 --- a/requests/defaults.py +++ b/requests/defaults.py @@ -6,15 +6,18 @@ requests.defaults This module provides the Requests configuration defaults. -settings parameters: +Configurations: -- :base_headers: - Sets default User-Agent to `python-requests.org` -- :accept_gzip: - Whether or not to accept gzip-compressed data -- :proxies: - http proxies? -- :verbose: - display verbose information? -- :timeout: - timeout time until request terminates -- :max_redirects: - maximum number of allowed redirects? -- :decode_unicode: - whether or not to accept unicode? +:base_headers: Default HTTP headers. +:verbose: Stream to write request logging to. +:timeout: Seconds until request timeout. +:max_redirects: Maximum njumber of redirects allowed within a request. +:decode_unicode: Decode unicode responses automatically? +:keep_alive: Reuse HTTP Connections? +:max_retries: The number of times a request should be retried in the event of a connection failure. +:safe_mode: If true, Requests will catch all errors. +:pool_maxsize: The maximium size of an HTTP connection pool. +:pool_connections: The number of active HTTP connection pools to use. """ @@ -29,31 +32,12 @@ defaults['base_headers'] = { 'Accept': '*/*' } - -#: Stream to log requests to. defaults['verbose'] = None - -#: Seconds until timeout. defaults['timeout'] = None - -#: Maximum number of redirects allowed within a request. defaults['max_redirects'] = 30 - -#: Should Requests decode unicode? defaults['decode_unicode'] = True - -#: Reuse HTTP Connections? defaults['keep_alive'] = True - -#: The number of active HTTP connection pools to use at a time. defaults['pool_connections'] = 10 - -#: The maximium size of an HTTP connection pool. defaults['pool_maxsize'] = 1 - -#: The number of times a request should be retried in the event of a -#: connection failure. defaults['max_retries'] = 0 - -#: If true, Requests will catch all errors. defaults['safe_mode'] = False From a0c3570640e6132f845d9012dc2b8ca63d546ac1 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 17:26:13 -0800 Subject: [PATCH 198/255] better stepping for elastic design --- docs/api.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/api.rst b/docs/api.rst index adbd53f7..56182d40 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -52,6 +52,7 @@ Exceptions .. autoexception:: TooManyRedirects +.. _configurations: Configurations -------------- @@ -59,6 +60,7 @@ Configurations .. automodule:: requests.defaults +.. _async: Async ----- From 5bc919a24f1a28bb0e47725977a6f5cf690a8d30 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 17:26:23 -0800 Subject: [PATCH 199/255] cross-references in documentation --- docs/user/advanced.rst | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index a6019ee9..d3e62dcd 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -43,10 +43,7 @@ Any dictionaries that you pass to a request method will be merged with the sessi Sometimes you'll want to omit session-level keys from a dict parameter. To do this, you simply set that key's value to ``None`` in the method-level parameter. It will automatically be omitted. -All values that are contained within a session are directly available to you: - - Session attibutes: - ``auth``, ``config``, ``cookies``, ``headers``, ``hooks``, ``keep_alive``, ``params``, ``proxies``, ``timeout`` +All values that are contained within a session are directly available to you. See the:ref:`Session API Docs ` to learn more. @@ -54,13 +51,13 @@ Configuring Requests -------------------- Sometimes you may want to configure a request to customize it's behavior. To do -this, you can pass in a ``config`` dictionary to a request or session. +this, you can pass in a ``config`` dictionary to a request or session. See the :ref:`Configuration API Docs ` to learn more. Keep-Alive ---------- -Excellent news — keep alive is 100% automatic within a session! Couldn't be easier. +Excellent news — thanks to urllib3. keep-alive is 100% automatic within a session! Any requests that you make within a session will automatically reuse the appropriate connection! If you'd like to disable keep-alive, you can simply set the ``keep_alive`` configuration to ``False``:: From 341ac9fe8a9112ba8d263528e0dc26c84263f908 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 17:26:32 -0800 Subject: [PATCH 200/255] 800px yo! --- docs/_themes/kr/static/flasky.css_t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_themes/kr/static/flasky.css_t b/docs/_themes/kr/static/flasky.css_t index bb00e935..44172654 100644 --- a/docs/_themes/kr/static/flasky.css_t +++ b/docs/_themes/kr/static/flasky.css_t @@ -387,7 +387,7 @@ a:hover tt { } -@media screen and (max-width: 600px) { +@media screen and (max-width: 800px) { div.sphinxsidebar { display: none; From 02ea563a063f11c7c0bff1545ecb42402566081d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 17:30:03 -0800 Subject: [PATCH 201/255] massive css improvements --- docs/_themes/kr/static/flasky.css_t | 92 ++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/docs/_themes/kr/static/flasky.css_t b/docs/_themes/kr/static/flasky.css_t index 44172654..bf2a2664 100644 --- a/docs/_themes/kr/static/flasky.css_t +++ b/docs/_themes/kr/static/flasky.css_t @@ -387,7 +387,7 @@ a:hover tt { } -@media screen and (max-width: 800px) { +@media screen and (max-width: 870px) { div.sphinxsidebar { display: none; @@ -436,6 +436,96 @@ a:hover tt { display: none; } + + +} + + + +@media screen and (max-width: 875px) { + + body { + margin: 0; + padding: 20px 30px; + } + + div.documentwrapper { + float: none; + background: white; + } + + div.sphinxsidebar { + display: block; + float: none; + width: 102.5%; + margin: 50px -30px -20px -30px; + padding: 10px 20px; + background: #333; + color: white; + } + + div.sphinxsidebar h3, div.sphinxsidebar h4, div.sphinxsidebar p, + div.sphinxsidebar h3 a { + color: white; + } + + div.sphinxsidebar a { + color: #aaa; + } + + div.sphinxsidebar p.logo { + display: none; + } + + div.document { + width: 100%; + margin: 0; + } + + div.related { + display: block; + margin: 0; + padding: 10px 0 20px 0; + } + + div.related ul, + div.related ul li { + margin: 0; + padding: 0; + } + + div.footer { + display: none; + } + + div.bodywrapper { + margin: 0; + } + + div.body { + min-height: 0; + padding: 0; + } + + .rtd_doc_footer { + display: none; + } + + .document { + width: auto; + } + + .footer { + width: auto; + } + + .footer { + width: auto; + } + + .github { + display: none; + } } From f7257eada6ed60e5a09eb09487f3a71ee982f015 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 17:30:20 -0800 Subject: [PATCH 202/255] Session at root --- requests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/__init__.py b/requests/__init__.py index eb6639f9..027ea3c5 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -25,7 +25,7 @@ __copyright__ = 'Copyright 2011 Kenneth Reitz' from . import utils from .models import Request, Response from .api import request, get, head, post, patch, put, delete -from .sessions import session +from .sessions import session, Session from .status_codes import codes from .exceptions import ( RequestException, Timeout, URLRequired, From 4002efb56a0aca6c6a0281fba69bf45faaed4e88 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 17:34:21 -0800 Subject: [PATCH 203/255] history --- HISTORY.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 36d47f7c..c820a370 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,17 @@ History ------- + +0.8.0 (2011-11-09) +++++++++++++++++++ + +* Keep-alive support! +* Complete removal of Urllib2 +* Complete removal of Poster +* Complete removal of CookieJars +* New ConnectionError raising +* Safe_mode for error catching + 0.7.6 (2011-11-07) ++++++++++++++++++ From dfbfb2303fbc19c07a4a0141af1d70857ef0d44f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 17:34:59 -0800 Subject: [PATCH 204/255] ref --- docs/api.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/api.rst b/docs/api.rst index 56182d40..f77fc0ad 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -137,6 +137,8 @@ Classes .. autoclass:: requests.Request :inherited-members: +.. _sessionapi: + .. autoclass:: requests.Session :inherited-members: From bd45bc716ac7cb7d7c0be369c655976d11d46cf0 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 17:46:19 -0800 Subject: [PATCH 205/255] add prefetch to main request distpatch --- requests/models.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/requests/models.py b/requests/models.py index d871e719..1deff7f2 100644 --- a/requests/models.py +++ b/requests/models.py @@ -304,7 +304,7 @@ class Request(object): return self.url - def send(self, anyway=False): + def send(self, anyway=False, prefetch=False): """Sends the request. Returns True of successful, false if not. If there was an HTTPError during transmission, self.response.status_code will contain the HTTPError code. @@ -364,11 +364,17 @@ class Request(object): # Update self to reflect the auth changes. self.__dict__.update(r.__dict__) - # Check to see if keep_alive is allowed. - if self.config.get('keep_alive'): - conn = self._poolmanager.connection_from_url(url) + _p = urlparse(url) + proxy = self.proxies.get(_p.scheme) + + if proxy: + conn = connectionpool.proxy_from_url() else: - conn = connectionpool.connection_from_url(url) + # Check to see if keep_alive is allowed. + if self.config.get('keep_alive'): + conn = self._poolmanager.connection_from_url(url) + else: + conn = connectionpool.connection_from_url(url) if not self.sent or anyway: @@ -397,7 +403,7 @@ class Request(object): headers=self.headers, redirect=False, assert_same_host=False, - preload_content=False, + preload_content=prefetch, decode_content=False, retries=self.config.get('max_retries', 0) ) From 0c4342a3a67834ce99ad73a7f3a6a0fe1780b19b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 17:47:04 -0800 Subject: [PATCH 206/255] prefetching on all methods --- requests/api.py | 3 ++- requests/sessions.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/requests/api.py b/requests/api.py index a139f3d5..bc9fde7e 100644 --- a/requests/api.py +++ b/requests/api.py @@ -28,6 +28,7 @@ def request(method, url, proxies=None, hooks=None, return_response=True, + prefetch=False, config=None): """Constructs and sends a :class:`Request `. Returns :class:`Response ` object. @@ -51,7 +52,7 @@ def request(method, url, return s.request( method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, return_response, - config + config, prefetch ) diff --git a/requests/sessions.py b/requests/sessions.py index ddac7860..83bfe129 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -112,7 +112,8 @@ class Session(object): proxies=None, hooks=None, return_response=True, - config=None): + config=None, + prefetch=False): """Constructs and sends a :class:`Request `. Returns :class:`Response ` object. @@ -130,6 +131,7 @@ class Session(object): :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. :param return_response: (optional) If False, an un-sent Request object will returned. :param config: (optional) A configuration dictionary. + :param prefetch: (optional) if ``True``, the response content will be immediately downloaded. """ method = str(method).upper() @@ -177,7 +179,7 @@ class Session(object): return r # Send the HTTP Request. - r.send() + r.send(prefetch=prefetch) # Send any cookies back up the to the session. self.cookies.update(r.response.cookies) From 87239802c143b2aeb935d738f6b3e2a1ad856be7 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 17:50:36 -0800 Subject: [PATCH 207/255] proxy pooling! --- requests/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index 1deff7f2..c803c126 100644 --- a/requests/models.py +++ b/requests/models.py @@ -20,7 +20,7 @@ from .hooks import dispatch_hook from .structures import CaseInsensitiveDict from .status_codes import codes from .packages.urllib3.exceptions import MaxRetryError -from .packages.urllib3 import connectionpool +from .packages.urllib3 import connectionpool, poolmanager from .exceptions import ( Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError) from .utils import ( @@ -368,7 +368,7 @@ class Request(object): proxy = self.proxies.get(_p.scheme) if proxy: - conn = connectionpool.proxy_from_url() + conn = poolmanager.proxy_from_url(url) else: # Check to see if keep_alive is allowed. if self.config.get('keep_alive'): From 0b424e1637ea552b4b653795838c5f747aa989f5 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 17:52:20 -0800 Subject: [PATCH 208/255] note about prefetch method --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index c820a370..18d4d6bb 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,7 @@ History * Complete removal of CookieJars * New ConnectionError raising * Safe_mode for error catching +* prefetch parameter for request methods 0.7.6 (2011-11-07) ++++++++++++++++++ From 2999c71e468a589595c4dfd01f8167f544dbe111 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 17:54:13 -0800 Subject: [PATCH 209/255] more proper prefetch --- requests/async.py | 1 + 1 file changed, 1 insertion(+) diff --git a/requests/async.py b/requests/async.py index 92839c30..84abd40d 100644 --- a/requests/async.py +++ b/requests/async.py @@ -34,6 +34,7 @@ def patched(f): def wrapped(*args, **kwargs): kwargs['return_response'] = False + kwargs['prefetch'] = True return f(*args, **kwargs) From d1b6db05a4aa48e10f08015889da0dfb20f4dc4c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 17:56:29 -0800 Subject: [PATCH 210/255] prefetch handling for _content_consumed --- requests/models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requests/models.py b/requests/models.py index c803c126..7825956e 100644 --- a/requests/models.py +++ b/requests/models.py @@ -423,6 +423,10 @@ class Request(object): r = dispatch_hook('post_request', self.hooks, self) self.__dict__.update(r.__dict__) + # If prefetch is True, mark content as consumed. + if prefetch: + self.response._content_consumed = True + return self.sent From b17ec8e1df8257c2756a779c27ccc5cc43b38a24 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 19:09:11 -0800 Subject: [PATCH 211/255] yep! --- docs/community/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/community/faq.rst b/docs/community/faq.rst index dd6f80d9..bad71b88 100644 --- a/docs/community/faq.rst +++ b/docs/community/faq.rst @@ -73,7 +73,7 @@ Support for Python 3.x is coming *very* soon. Keep-alive Support? ------------------- -It's on the way. +Yep! Proxy Support? From fad1c665a640701c588924f8c9606f69b0662322 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 21:36:34 -0800 Subject: [PATCH 212/255] remove the jar references --- docs/user/advanced.rst | 4 ++-- docs/user/quickstart.rst | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index d3e62dcd..34c934a3 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -10,8 +10,8 @@ Session Objects --------------- The Session object allows you to persist certain parameters across -requests. It also establishes a CookieJar and passes it along -to any requests made from the Session instance. +requests. It also perstists cookies across all requests made from the +Session instance. A session object has all the methods of the main Requests API. diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 17e503a2..93feb4d7 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -122,11 +122,6 @@ If a response contains some Cookies, you can get quick access to them:: >>> print r.cookies {'requests-is': 'awesome'} -The underlying CookieJar is also available for more advanced handling:: - - >>> r.request.cookiejar - - To send your own cookies to the server, you can use the ``cookies`` parameter:: From f4296ba758e9bfd598966c7f2ec988ec305c9ebe Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 22:01:24 -0800 Subject: [PATCH 213/255] Joseph McCullough --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 131c0f61..776e791c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -49,4 +49,5 @@ Patches and Suggestions - Daniel Hengeveld - Dan Head - Bruno Renié -- David Fischer \ No newline at end of file +- David Fischer +- Joseph McCullough \ No newline at end of file From ae9cdf1a478bebc1a5e0afc49ebb81d2ca3aa544 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 22:18:35 -0800 Subject: [PATCH 214/255] options history --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index 18d4d6bb..8349b5c5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,7 @@ History * New ConnectionError raising * Safe_mode for error catching * prefetch parameter for request methods +* OPTION method 0.7.6 (2011-11-07) ++++++++++++++++++ From 1588ad7c469b61cbfc8185e3d7326dff79afa81e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 9 Nov 2011 22:44:58 -0800 Subject: [PATCH 215/255] remove poster --- requests/packages/poster/__init__.py | 34 -- requests/packages/poster/encode.py | 414 ---------------------- requests/packages/poster/streaminghttp.py | 199 ----------- 3 files changed, 647 deletions(-) delete mode 100644 requests/packages/poster/__init__.py delete mode 100644 requests/packages/poster/encode.py delete mode 100644 requests/packages/poster/streaminghttp.py diff --git a/requests/packages/poster/__init__.py b/requests/packages/poster/__init__.py deleted file mode 100644 index 6e216fce..00000000 --- a/requests/packages/poster/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2010 Chris AtLee -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -"""poster module - -Support for streaming HTTP uploads, and multipart/form-data encoding - -```poster.version``` is a 3-tuple of integers representing the version number. -New releases of poster will always have a version number that compares greater -than an older version of poster. -New in version 0.6.""" - -from __future__ import absolute_import - -from . import streaminghttp -from . import encode - -version = (0, 8, 0) # Thanks JP! diff --git a/requests/packages/poster/encode.py b/requests/packages/poster/encode.py deleted file mode 100644 index cf2298d7..00000000 --- a/requests/packages/poster/encode.py +++ /dev/null @@ -1,414 +0,0 @@ -"""multipart/form-data encoding module - -This module provides functions that faciliate encoding name/value pairs -as multipart/form-data suitable for a HTTP POST or PUT request. - -multipart/form-data is the standard way to upload files over HTTP""" - -__all__ = ['gen_boundary', 'encode_and_quote', 'MultipartParam', - 'encode_string', 'encode_file_header', 'get_body_size', 'get_headers', - 'multipart_encode'] - -try: - import uuid - def gen_boundary(): - """Returns a random string to use as the boundary for a message""" - return uuid.uuid4().hex -except ImportError: - import random, sha - def gen_boundary(): - """Returns a random string to use as the boundary for a message""" - bits = random.getrandbits(160) - return sha.new(str(bits)).hexdigest() - -import urllib, re, os, mimetypes -try: - from email.header import Header -except ImportError: - # Python 2.4 - from email.Header import Header - -def encode_and_quote(data): - """If ``data`` is unicode, return urllib.quote_plus(data.encode("utf-8")) - otherwise return urllib.quote_plus(data)""" - if data is None: - return None - - if isinstance(data, unicode): - data = data.encode("utf-8") - return urllib.quote_plus(data) - -def _strify(s): - """If s is a unicode string, encode it to UTF-8 and return the results, - otherwise return str(s), or None if s is None""" - if s is None: - return None - if isinstance(s, unicode): - return s.encode("utf-8") - return str(s) - -class MultipartParam(object): - """Represents a single parameter in a multipart/form-data request - - ``name`` is the name of this parameter. - - If ``value`` is set, it must be a string or unicode object to use as the - data for this parameter. - - If ``filename`` is set, it is what to say that this parameter's filename - is. Note that this does not have to be the actual filename any local file. - - If ``filetype`` is set, it is used as the Content-Type for this parameter. - If unset it defaults to "text/plain; charset=utf8" - - If ``filesize`` is set, it specifies the length of the file ``fileobj`` - - If ``fileobj`` is set, it must be a file-like object that supports - .read(). - - Both ``value`` and ``fileobj`` must not be set, doing so will - raise a ValueError assertion. - - If ``fileobj`` is set, and ``filesize`` is not specified, then - the file's size will be determined first by stat'ing ``fileobj``'s - file descriptor, and if that fails, by seeking to the end of the file, - recording the current position as the size, and then by seeking back to the - beginning of the file. - - ``cb`` is a callable which will be called from iter_encode with (self, - current, total), representing the current parameter, current amount - transferred, and the total size. - """ - def __init__(self, name, value=None, filename=None, filetype=None, - filesize=None, fileobj=None, cb=None): - self.name = Header(name).encode() - self.value = _strify(value) - if filename is None: - self.filename = None - else: - if isinstance(filename, unicode): - # Encode with XML entities - self.filename = filename.encode("ascii", "xmlcharrefreplace") - else: - self.filename = str(filename) - self.filename = self.filename.encode("string_escape").\ - replace('"', '\\"') - self.filetype = _strify(filetype) - - self.filesize = filesize - self.fileobj = fileobj - self.cb = cb - - if self.value is not None and self.fileobj is not None: - raise ValueError("Only one of value or fileobj may be specified") - - if fileobj is not None and filesize is None: - # Try and determine the file size - try: - self.filesize = os.fstat(fileobj.fileno()).st_size - except (OSError, AttributeError): - try: - fileobj.seek(0, 2) - self.filesize = fileobj.tell() - fileobj.seek(0) - except: - raise ValueError("Could not determine filesize") - - def __cmp__(self, other): - attrs = ['name', 'value', 'filename', 'filetype', 'filesize', 'fileobj'] - myattrs = [getattr(self, a) for a in attrs] - oattrs = [getattr(other, a) for a in attrs] - return cmp(myattrs, oattrs) - - def reset(self): - if self.fileobj is not None: - self.fileobj.seek(0) - elif self.value is None: - raise ValueError("Don't know how to reset this parameter") - - @classmethod - def from_file(cls, paramname, filename): - """Returns a new MultipartParam object constructed from the local - file at ``filename``. - - ``filesize`` is determined by os.path.getsize(``filename``) - - ``filetype`` is determined by mimetypes.guess_type(``filename``)[0] - - ``filename`` is set to os.path.basename(``filename``) - """ - - return cls(paramname, filename=os.path.basename(filename), - filetype=mimetypes.guess_type(filename)[0], - filesize=os.path.getsize(filename), - fileobj=open(filename, "rb")) - - @classmethod - def from_params(cls, params): - """Returns a list of MultipartParam objects from a sequence of - name, value pairs, MultipartParam instances, - or from a mapping of names to values - - The values may be strings or file objects, or MultipartParam objects. - MultipartParam object names must match the given names in the - name,value pairs or mapping, if applicable.""" - if hasattr(params, 'items'): - params = params.items() - - retval = [] - for item in params: - if isinstance(item, cls): - retval.append(item) - continue - name, value = item - if isinstance(value, cls): - assert value.name == name - retval.append(value) - continue - if hasattr(value, 'read'): - # Looks like a file object - filename = getattr(value, 'name', None) - if filename is not None: - filetype = mimetypes.guess_type(filename)[0] - else: - filetype = None - - retval.append(cls(name=name, filename=filename, - filetype=filetype, fileobj=value)) - else: - retval.append(cls(name, value)) - return retval - - def encode_hdr(self, boundary): - """Returns the header of the encoding of this parameter""" - boundary = encode_and_quote(boundary) - - headers = ["--%s" % boundary] - - if self.filename: - disposition = 'form-data; name="%s"; filename="%s"' % (self.name, - self.filename) - else: - disposition = 'form-data; name="%s"' % self.name - - headers.append("Content-Disposition: %s" % disposition) - - if self.filetype: - filetype = self.filetype - else: - filetype = "text/plain; charset=utf-8" - - headers.append("Content-Type: %s" % filetype) - - headers.append("") - headers.append("") - - return "\r\n".join(headers) - - def encode(self, boundary): - """Returns the string encoding of this parameter""" - if self.value is None: - value = self.fileobj.read() - else: - value = self.value - - if re.search("^--%s$" % re.escape(boundary), value, re.M): - raise ValueError("boundary found in encoded string") - - return "%s%s\r\n" % (self.encode_hdr(boundary), value) - - def iter_encode(self, boundary, blocksize=4096): - """Yields the encoding of this parameter - If self.fileobj is set, then blocks of ``blocksize`` bytes are read and - yielded.""" - total = self.get_size(boundary) - current = 0 - if self.value is not None: - block = self.encode(boundary) - current += len(block) - yield block - if self.cb: - self.cb(self, current, total) - else: - block = self.encode_hdr(boundary) - current += len(block) - yield block - if self.cb: - self.cb(self, current, total) - last_block = "" - encoded_boundary = "--%s" % encode_and_quote(boundary) - boundary_exp = re.compile("^%s$" % re.escape(encoded_boundary), - re.M) - while True: - block = self.fileobj.read(blocksize) - if not block: - current += 2 - yield "\r\n" - if self.cb: - self.cb(self, current, total) - break - last_block += block - if boundary_exp.search(last_block): - raise ValueError("boundary found in file data") - last_block = last_block[-len(encoded_boundary)-2:] - current += len(block) - yield block - if self.cb: - self.cb(self, current, total) - - def get_size(self, boundary): - """Returns the size in bytes that this param will be when encoded - with the given boundary.""" - if self.filesize is not None: - valuesize = self.filesize - else: - valuesize = len(self.value) - - return len(self.encode_hdr(boundary)) + 2 + valuesize - -def encode_string(boundary, name, value): - """Returns ``name`` and ``value`` encoded as a multipart/form-data - variable. ``boundary`` is the boundary string used throughout - a single request to separate variables.""" - - return MultipartParam(name, value).encode(boundary) - -def encode_file_header(boundary, paramname, filesize, filename=None, - filetype=None): - """Returns the leading data for a multipart/form-data field that contains - file data. - - ``boundary`` is the boundary string used throughout a single request to - separate variables. - - ``paramname`` is the name of the variable in this request. - - ``filesize`` is the size of the file data. - - ``filename`` if specified is the filename to give to this field. This - field is only useful to the server for determining the original filename. - - ``filetype`` if specified is the MIME type of this file. - - The actual file data should be sent after this header has been sent. - """ - - return MultipartParam(paramname, filesize=filesize, filename=filename, - filetype=filetype).encode_hdr(boundary) - -def get_body_size(params, boundary): - """Returns the number of bytes that the multipart/form-data encoding - of ``params`` will be.""" - size = sum(p.get_size(boundary) for p in MultipartParam.from_params(params)) - return size + len(boundary) + 6 - -def get_headers(params, boundary): - """Returns a dictionary with Content-Type and Content-Length headers - for the multipart/form-data encoding of ``params``.""" - headers = {} - boundary = urllib.quote_plus(boundary) - headers['Content-Type'] = "multipart/form-data; boundary=%s" % boundary - headers['Content-Length'] = str(get_body_size(params, boundary)) - return headers - -class multipart_yielder: - def __init__(self, params, boundary, cb): - self.params = params - self.boundary = boundary - self.cb = cb - - self.i = 0 - self.p = None - self.param_iter = None - self.current = 0 - self.total = get_body_size(params, boundary) - - def __iter__(self): - return self - - def next(self): - """generator function to yield multipart/form-data representation - of parameters""" - if self.param_iter is not None: - try: - block = self.param_iter.next() - self.current += len(block) - if self.cb: - self.cb(self.p, self.current, self.total) - return block - except StopIteration: - self.p = None - self.param_iter = None - - if self.i is None: - raise StopIteration - elif self.i >= len(self.params): - self.param_iter = None - self.p = None - self.i = None - block = "--%s--\r\n" % self.boundary - self.current += len(block) - if self.cb: - self.cb(self.p, self.current, self.total) - return block - - self.p = self.params[self.i] - self.param_iter = self.p.iter_encode(self.boundary) - self.i += 1 - return self.next() - - def reset(self): - self.i = 0 - self.current = 0 - for param in self.params: - param.reset() - -def multipart_encode(params, boundary=None, cb=None): - """Encode ``params`` as multipart/form-data. - - ``params`` should be a sequence of (name, value) pairs or MultipartParam - objects, or a mapping of names to values. - Values are either strings parameter values, or file-like objects to use as - the parameter value. The file-like objects must support .read() and either - .fileno() or both .seek() and .tell(). - - If ``boundary`` is set, then it as used as the MIME boundary. Otherwise - a randomly generated boundary will be used. In either case, if the - boundary string appears in the parameter values a ValueError will be - raised. - - If ``cb`` is set, it should be a callback which will get called as blocks - of data are encoded. It will be called with (param, current, total), - indicating the current parameter being encoded, the current amount encoded, - and the total amount to encode. - - Returns a tuple of `datagen`, `headers`, where `datagen` is a - generator that will yield blocks of data that make up the encoded - parameters, and `headers` is a dictionary with the assoicated - Content-Type and Content-Length headers. - - Examples: - - >>> datagen, headers = multipart_encode( [("key", "value1"), ("key", "value2")] ) - >>> s = "".join(datagen) - >>> assert "value2" in s and "value1" in s - - >>> p = MultipartParam("key", "value2") - >>> datagen, headers = multipart_encode( [("key", "value1"), p] ) - >>> s = "".join(datagen) - >>> assert "value2" in s and "value1" in s - - >>> datagen, headers = multipart_encode( {"key": "value1"} ) - >>> s = "".join(datagen) - >>> assert "value2" not in s and "value1" in s - - """ - if boundary is None: - boundary = gen_boundary() - else: - boundary = urllib.quote_plus(boundary) - - headers = get_headers(params, boundary) - params = MultipartParam.from_params(params) - - return multipart_yielder(params, boundary, cb), headers diff --git a/requests/packages/poster/streaminghttp.py b/requests/packages/poster/streaminghttp.py deleted file mode 100644 index 1b591d4b..00000000 --- a/requests/packages/poster/streaminghttp.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Streaming HTTP uploads module. - -This module extends the standard httplib and urllib2 objects so that -iterable objects can be used in the body of HTTP requests. - -In most cases all one should have to do is call :func:`register_openers()` -to register the new streaming http handlers which will take priority over -the default handlers, and then you can use iterable objects in the body -of HTTP requests. - -**N.B.** You must specify a Content-Length header if using an iterable object -since there is no way to determine in advance the total size that will be -yielded, and there is no way to reset an interator. - -Example usage: - ->>> from StringIO import StringIO ->>> import urllib2, poster.streaminghttp - ->>> opener = poster.streaminghttp.register_openers() - ->>> s = "Test file data" ->>> f = StringIO(s) - ->>> req = urllib2.Request("http://localhost:5000", f, -... {'Content-Length': str(len(s))}) -""" - -import httplib, urllib2, socket -from httplib import NotConnected - -__all__ = ['StreamingHTTPConnection', 'StreamingHTTPRedirectHandler', - 'StreamingHTTPHandler', 'register_openers'] - -if hasattr(httplib, 'HTTPS'): - __all__.extend(['StreamingHTTPSHandler', 'StreamingHTTPSConnection']) - -class _StreamingHTTPMixin: - """Mixin class for HTTP and HTTPS connections that implements a streaming - send method.""" - def send(self, value): - """Send ``value`` to the server. - - ``value`` can be a string object, a file-like object that supports - a .read() method, or an iterable object that supports a .next() - method. - """ - # Based on python 2.6's httplib.HTTPConnection.send() - if self.sock is None: - if self.auto_open: - self.connect() - else: - raise NotConnected() - - # send the data to the server. if we get a broken pipe, then close - # the socket. we want to reconnect when somebody tries to send again. - # - # NOTE: we DO propagate the error, though, because we cannot simply - # ignore the error... the caller will know if they can retry. - if self.debuglevel > 0: - print "send:", repr(value) - try: - blocksize = 8192 - if hasattr(value, 'read') : - if hasattr(value, 'seek'): - value.seek(0) - if self.debuglevel > 0: - print "sendIng a read()able" - data = value.read(blocksize) - while data: - self.sock.sendall(data) - data = value.read(blocksize) - elif hasattr(value, 'next'): - if hasattr(value, 'reset'): - value.reset() - if self.debuglevel > 0: - print "sendIng an iterable" - for data in value: - self.sock.sendall(data) - else: - self.sock.sendall(value) - except socket.error, v: - if v[0] == 32: # Broken pipe - self.close() - raise - -class StreamingHTTPConnection(_StreamingHTTPMixin, httplib.HTTPConnection): - """Subclass of `httplib.HTTPConnection` that overrides the `send()` method - to support iterable body objects""" - -class StreamingHTTPRedirectHandler(urllib2.HTTPRedirectHandler): - """Subclass of `urllib2.HTTPRedirectHandler` that overrides the - `redirect_request` method to properly handle redirected POST requests - - This class is required because python 2.5's HTTPRedirectHandler does - not remove the Content-Type or Content-Length headers when requesting - the new resource, but the body of the original request is not preserved. - """ - - handler_order = urllib2.HTTPRedirectHandler.handler_order - 1 - - # From python2.6 urllib2's HTTPRedirectHandler - def redirect_request(self, req, fp, code, msg, headers, newurl): - """Return a Request or None in response to a redirect. - - This is called by the http_error_30x methods when a - redirection response is received. If a redirection should - take place, return a new Request to allow http_error_30x to - perform the redirect. Otherwise, raise HTTPError if no-one - else should try to handle this url. Return None if you can't - but another Handler might. - """ - m = req.get_method() - if (code in (301, 302, 303, 307) and m in ("GET", "HEAD") - or code in (301, 302, 303) and m == "POST"): - # Strictly (according to RFC 2616), 301 or 302 in response - # to a POST MUST NOT cause a redirection without confirmation - # from the user (of urllib2, in this case). In practice, - # essentially all clients do redirect in this case, so we - # do the same. - # be conciliant with URIs containing a space - newurl = newurl.replace(' ', '%20') - newheaders = dict((k, v) for k, v in req.headers.items() - if k.lower() not in ( - "content-length", "content-type") - ) - return urllib2.Request(newurl, - headers=newheaders, - origin_req_host=req.get_origin_req_host(), - unverifiable=True) - else: - raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp) - -class StreamingHTTPHandler(urllib2.HTTPHandler): - """Subclass of `urllib2.HTTPHandler` that uses - StreamingHTTPConnection as its http connection class.""" - - handler_order = urllib2.HTTPHandler.handler_order - 1 - - def http_open(self, req): - """Open a StreamingHTTPConnection for the given request""" - return self.do_open(StreamingHTTPConnection, req) - - def http_request(self, req): - """Handle a HTTP request. Make sure that Content-Length is specified - if we're using an interable value""" - # Make sure that if we're using an iterable object as the request - # body, that we've also specified Content-Length - if req.has_data(): - data = req.get_data() - if hasattr(data, 'read') or hasattr(data, 'next'): - if not req.has_header('Content-length'): - raise ValueError( - "No Content-Length specified for iterable body") - return urllib2.HTTPHandler.do_request_(self, req) - -if hasattr(httplib, 'HTTPS'): - class StreamingHTTPSConnection(_StreamingHTTPMixin, - httplib.HTTPSConnection): - """Subclass of `httplib.HTTSConnection` that overrides the `send()` - method to support iterable body objects""" - - class StreamingHTTPSHandler(urllib2.HTTPSHandler): - """Subclass of `urllib2.HTTPSHandler` that uses - StreamingHTTPSConnection as its http connection class.""" - - handler_order = urllib2.HTTPSHandler.handler_order - 1 - - def https_open(self, req): - return self.do_open(StreamingHTTPSConnection, req) - - def https_request(self, req): - # Make sure that if we're using an iterable object as the request - # body, that we've also specified Content-Length - if req.has_data(): - data = req.get_data() - if hasattr(data, 'read') or hasattr(data, 'next'): - if not req.has_header('Content-length'): - raise ValueError( - "No Content-Length specified for iterable body") - return urllib2.HTTPSHandler.do_request_(self, req) - - -def get_handlers(): - handlers = [StreamingHTTPHandler, StreamingHTTPRedirectHandler] - if hasattr(httplib, "HTTPS"): - handlers.append(StreamingHTTPSHandler) - return handlers - -def register_openers(): - """Register the streaming http handlers in the global urllib2 default - opener object. - - Returns the created OpenerDirector object.""" - opener = urllib2.build_opener(*get_handlers()) - - urllib2.install_opener(opener) - - return opener From a348fd7ae3c6349b726410a75e0c551dadf4c1c9 Mon Sep 17 00:00:00 2001 From: Devin Sevilla Date: Thu, 10 Nov 2011 15:14:38 -0800 Subject: [PATCH 216/255] The 'requests-oauth-hook' project was renamed to requests-oauth --- docs/community/out-there.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/community/out-there.rst b/docs/community/out-there.rst index 7870de9e..553c7444 100644 --- a/docs/community/out-there.rst +++ b/docs/community/out-there.rst @@ -1,7 +1,7 @@ Modules ======= -- `requests-oauth-hook `_, adds OAuth support to Requests. +- `requests-oauth `_, adds OAuth support to Requests. - `FacePy `_, a Python wrapper to the Facebook API. - `robotframework-requests `_, a Robot Framework API wrapper. - `fullerene `_, a Graphite Dashboard. From 2d8440ea98301131edd537c4f2010b749f005e3b Mon Sep 17 00:00:00 2001 From: Devin Sevilla Date: Thu, 10 Nov 2011 15:34:18 -0800 Subject: [PATCH 217/255] Verbose logging example does not work --- docs/user/advanced.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 34c934a3..9083b430 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -227,7 +227,7 @@ by your application, you can turn on verbose logging. To do so, just configure Requests with a stream to write to:: - >>> requests.settings.verbose = sys.stderr - >>> requests.get('http://httpbin.org/headers') + >>> my_config = {'verbose': sys.stderr} + >>> requests.get('http://httpbin.org/headers', config=my_config) 2011-08-17T03:04:23.380175 GET http://httpbin.org/headers From afc6561fdbb0a9ca909dad72d6672c7525788f77 Mon Sep 17 00:00:00 2001 From: Joseph McCullough Date: Fri, 11 Nov 2011 22:17:22 -0600 Subject: [PATCH 218/255] Fixed session API ref typo --- docs/user/advanced.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 9083b430..df274994 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -43,7 +43,7 @@ Any dictionaries that you pass to a request method will be merged with the sessi Sometimes you'll want to omit session-level keys from a dict parameter. To do this, you simply set that key's value to ``None`` in the method-level parameter. It will automatically be omitted. -All values that are contained within a session are directly available to you. See the:ref:`Session API Docs ` to learn more. +All values that are contained within a session are directly available to you. See the :ref:`Session API Docs ` to learn more. From 6833b326f801272afc50d2a816147021c4c62057 Mon Sep 17 00:00:00 2001 From: Joseph McCullough Date: Fri, 11 Nov 2011 23:15:19 -0600 Subject: [PATCH 219/255] Fixed gevent reference --- docs/user/advanced.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index df274994..d4a416cf 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -72,7 +72,7 @@ Requests has first-class support for concurrent requests, powered by gevent. This allows you to send a bunch of HTTP requests at the same First, let's import the async module. Heads up — if you don't have -`gevent `_ this will fail:: +`gevent `_ this will fail:: from requests import async From f5f2ddfa747be428a66139644266e1cb7dcc3ae5 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 07:35:41 -0800 Subject: [PATCH 220/255] throttling --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index 8349b5c5..0648118d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,7 @@ History * Safe_mode for error catching * prefetch parameter for request methods * OPTION method +* Async pool size throttling 0.7.6 (2011-11-07) ++++++++++++++++++ From 69e7e94c4ce349b243d8dde82a69166d8792ab81 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 07:35:55 -0800 Subject: [PATCH 221/255] updated urllib3 --- requests/packages/__init__.py | 2 +- requests/packages/urllib3/__init__.py | 2 +- requests/packages/urllib3/_collections.py | 42 ++++++++++++------- requests/packages/urllib3/connectionpool.py | 0 requests/packages/urllib3/contrib/__init__.py | 0 requests/packages/urllib3/contrib/ntlmpool.py | 0 requests/packages/urllib3/exceptions.py | 0 requests/packages/urllib3/filepost.py | 0 requests/packages/urllib3/poolmanager.py | 2 +- requests/packages/urllib3/request.py | 0 requests/packages/urllib3/response.py | 0 11 files changed, 31 insertions(+), 17 deletions(-) mode change 100644 => 100755 requests/packages/urllib3/__init__.py mode change 100644 => 100755 requests/packages/urllib3/_collections.py mode change 100644 => 100755 requests/packages/urllib3/connectionpool.py mode change 100644 => 100755 requests/packages/urllib3/contrib/__init__.py mode change 100644 => 100755 requests/packages/urllib3/contrib/ntlmpool.py mode change 100644 => 100755 requests/packages/urllib3/exceptions.py mode change 100644 => 100755 requests/packages/urllib3/filepost.py mode change 100644 => 100755 requests/packages/urllib3/poolmanager.py mode change 100644 => 100755 requests/packages/urllib3/request.py mode change 100644 => 100755 requests/packages/urllib3/response.py diff --git a/requests/packages/__init__.py b/requests/packages/__init__.py index ab2669e8..d62c4b71 100644 --- a/requests/packages/__init__.py +++ b/requests/packages/__init__.py @@ -1,3 +1,3 @@ from __future__ import absolute_import -from . import poster +from . import urllib3 diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py old mode 100644 new mode 100755 index 7a2d0186..20b1fb4e --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -10,7 +10,7 @@ urllib3 - Thread-safe connection pooling and re-using. __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.0.1' +__version__ = '1.0.2' from .connectionpool import ( diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py old mode 100644 new mode 100755 index bd01da33..e73f0ed4 --- a/requests/packages/urllib3/_collections.py +++ b/requests/packages/urllib3/_collections.py @@ -4,8 +4,9 @@ # This module is part of urllib3 and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from collections import MutableMapping, deque +from collections import deque, MutableMapping +from threading import RLock __all__ = ['RecentlyUsedContainer'] @@ -24,9 +25,6 @@ class RecentlyUsedContainer(MutableMapping): away the least-recently-used keys beyond ``maxsize``. """ - # TODO: Make this threadsafe. _prune_invalidated_entries should be the - # only real pain-point for this. - # If len(self.access_log) exceeds self._maxsize * CLEANUP_FACTOR, then we # will attempt to cleanup the invalidated entries in the access_log # datastructure during the next 'get' operation. @@ -39,6 +37,7 @@ class RecentlyUsedContainer(MutableMapping): # We use a deque to to store our keys ordered by the last access. self.access_log = deque() + self.access_log_lock = RLock() # We look up the access log entry by the key to invalidate it so we can # insert a new authorative entry at the head without having to dig and @@ -48,43 +47,58 @@ class RecentlyUsedContainer(MutableMapping): # Trigger a heap cleanup when we get past this size self.access_log_limit = maxsize * self.CLEANUP_FACTOR - def _push_entry(self, key): - "Push entry onto our access log, invalidate the old entry if exists." - # Invalidate old entry if it exists + def _invalidate_entry(self, key): + "If exists: Invalidate old entry and return it." old_entry = self.access_lookup.get(key) if old_entry: old_entry.is_valid = False - new_entry = AccessEntry(key) + return old_entry + def _push_entry(self, key): + "Push entry onto our access log, invalidate the old entry if exists." + self._invalidate_entry(key) + + new_entry = AccessEntry(key) self.access_lookup[key] = new_entry + + self.access_log_lock.acquire() self.access_log.appendleft(new_entry) + self.access_log_lock.release() def _prune_entries(self, num): "Pop entries from our access log until we popped ``num`` valid ones." while num > 0: + self.access_log_lock.acquire() p = self.access_log.pop() + self.access_log_lock.release() if not p.is_valid: continue # Invalidated entry, skip - del self._container[p.key] - del self.access_lookup[p.key] + self._container.pop(p.key, None) + self.access_lookup.pop(p.key, None) num -= 1 def _prune_invalidated_entries(self): "Rebuild our access_log without the invalidated entries." + self.access_log_lock.acquire() self.access_log = deque(e for e in self.access_log if e.is_valid) + self.access_log_lock.release() def _get_ordered_access_keys(self): - # Used for testing - return [e.key for e in self.access_log if e.is_valid] + "Return ordered access keys for inspection. Used for testing." + self.access_log_lock.acquire() + r = [e.key for e in self.access_log if e.is_valid] + self.access_log_lock.release() + + return r def __getitem__(self, key): item = self._container.get(key) if not item: - return + raise KeyError(key) # Insert new entry with new high priority, also implicitly invalidates # the old entry. @@ -108,7 +122,7 @@ class RecentlyUsedContainer(MutableMapping): def __delitem__(self, key): self._invalidate_entry(key) del self._container[key] - del self._access_lookup[key] + del self.access_lookup[key] def __len__(self): return self._container.__len__() diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py old mode 100644 new mode 100755 diff --git a/requests/packages/urllib3/contrib/__init__.py b/requests/packages/urllib3/contrib/__init__.py old mode 100644 new mode 100755 diff --git a/requests/packages/urllib3/contrib/ntlmpool.py b/requests/packages/urllib3/contrib/ntlmpool.py old mode 100644 new mode 100755 diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py old mode 100644 new mode 100755 diff --git a/requests/packages/urllib3/filepost.py b/requests/packages/urllib3/filepost.py old mode 100644 new mode 100755 diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py old mode 100644 new mode 100755 index 622789e5..c08e327f --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -108,7 +108,7 @@ class PoolManager(RequestMethods): return conn.urlopen(method, url, assert_same_host=False, **kw) -class ProxyManager(object): +class ProxyManager(RequestMethods): """ Given a ConnectionPool to a proxy, the ProxyManager's ``urlopen`` method will make requests to any url through the defined proxy. diff --git a/requests/packages/urllib3/request.py b/requests/packages/urllib3/request.py old mode 100644 new mode 100755 diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py old mode 100644 new mode 100755 From cdd0613aecba646f431b1a4e9164aa7b9d8e24be Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 07:45:34 -0800 Subject: [PATCH 222/255] Update AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index e3dd319f..20756691 100644 --- a/AUTHORS +++ b/AUTHORS @@ -52,3 +52,4 @@ Patches and Suggestions - David Fischer - Joseph McCullough - Juergen Brendel +- Juan Riaza From 50279dd83c378e2de73695db9ebe06a2ba079a1f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 10:00:30 -0800 Subject: [PATCH 223/255] Remove timeout configuration --- requests/defaults.py | 1 - 1 file changed, 1 deletion(-) diff --git a/requests/defaults.py b/requests/defaults.py index e3f3ba37..eee4ae59 100644 --- a/requests/defaults.py +++ b/requests/defaults.py @@ -33,7 +33,6 @@ defaults['base_headers'] = { } defaults['verbose'] = None -defaults['timeout'] = None defaults['max_redirects'] = 30 defaults['decode_unicode'] = True defaults['keep_alive'] = True From 7c206c74ba270bdfa2e748c33e4c6c89deea517b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 10:01:05 -0800 Subject: [PATCH 224/255] Timeouts for urllib3 --- requests/models.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 7825956e..64641c56 100644 --- a/requests/models.py +++ b/requests/models.py @@ -20,6 +20,8 @@ from .hooks import dispatch_hook from .structures import CaseInsensitiveDict from .status_codes import codes from .packages.urllib3.exceptions import MaxRetryError +from .packages.urllib3.exceptions import SSLError as _SSLError +from .packages.urllib3.exceptions import HTTPError as _HTTPError from .packages.urllib3 import connectionpool, poolmanager from .exceptions import ( Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError) @@ -405,15 +407,21 @@ class Request(object): assert_same_host=False, preload_content=prefetch, decode_content=False, - retries=self.config.get('max_retries', 0) + retries=self.config.get('max_retries', 0), + timeout=self.timeout, ) + except MaxRetryError, e: if not self.config.get('safe_mode', False): raise ConnectionError(e) else: r = None + except (_SSLError, _HTTPError), e: + if not self.config.get('safe_mode', False): + raise Timeout('Request timed out.') + self._build_response(r) # Response manipulation hook. From 52b9f01e6eec430177ea160fa068dfb0a6f215be Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 11:30:24 -0800 Subject: [PATCH 225/255] kill small css file --- docs/_themes/kr/layout.html | 2 - docs/_themes/kr/static/small_flask.css | 90 -------------------------- 2 files changed, 92 deletions(-) delete mode 100644 docs/_themes/kr/static/small_flask.css diff --git a/docs/_themes/kr/layout.html b/docs/_themes/kr/layout.html index 4a7affe2..7a9c7e0c 100644 --- a/docs/_themes/kr/layout.html +++ b/docs/_themes/kr/layout.html @@ -4,8 +4,6 @@ {% if theme_touch_icon %} {% endif %} - {% endblock %} {%- block relbar2 %}{% endblock %} diff --git a/docs/_themes/kr/static/small_flask.css b/docs/_themes/kr/static/small_flask.css deleted file mode 100644 index 8d55e95f..00000000 --- a/docs/_themes/kr/static/small_flask.css +++ /dev/null @@ -1,90 +0,0 @@ -/* - * small_flask.css_t - * ~~~~~~~~~~~~~~~~~ - * - * :copyright: Copyright 2010 by Armin Ronacher. - * :license: Flask Design License, see LICENSE for details. - */ - -body { - margin: 0; - padding: 20px 30px; -} - -div.documentwrapper { - float: none; - background: white; -} - -div.sphinxsidebar { - display: block; - float: none; - width: 102.5%; - margin: 50px -30px -20px -30px; - padding: 10px 20px; - background: #333; - color: white; -} - -div.sphinxsidebar h3, div.sphinxsidebar h4, div.sphinxsidebar p, -div.sphinxsidebar h3 a { - color: white; -} - -div.sphinxsidebar a { - color: #aaa; -} - -div.sphinxsidebar p.logo { - display: none; -} - -div.document { - width: 100%; - margin: 0; -} - -div.related { - display: block; - margin: 0; - padding: 10px 0 20px 0; -} - -div.related ul, -div.related ul li { - margin: 0; - padding: 0; -} - -div.footer { - display: none; -} - -div.bodywrapper { - margin: 0; -} - -div.body { - min-height: 0; - padding: 0; -} - -.rtd_doc_footer { - display: none; -} - -.document { - width: auto; -} - -.footer { - width: auto; -} - -.footer { - width: auto; -} - -.github { - display: none; -} \ No newline at end of file From 60558fdcc454baeb864868c01c5c43b2d1b774f1 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 12:39:31 -0800 Subject: [PATCH 226/255] big quick start updates --- docs/user/quickstart.rst | 88 ++++++++++++++++++++++++++++++++-------- 1 file changed, 72 insertions(+), 16 deletions(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index f4731f37..7148fe6a 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -39,25 +39,83 @@ We can read the content of the server's response:: >>> r.content '[{"repository":{"open_issues":0,"url":"https://github.com/... +Requests does its best to decode content from the server. Most unicode charsets, ``gzip``, and ``deflate`` encodings are all seamlessly decoded. + Make a POST Request ------------------- +------------------- -POST requests are equally simple :: +POST requests are equally simple:: - >>> r = requests.post("http://httpbin.org/post") - + r = requests.post("http://httpbin.org/post") -Suppose you want to send data over HTTP. Simply pass a data -argument to the requests.post method with your dictionary :: - >>> dataDict = {"key1":"value1", "key2":"value2"} - >>> r = requests.post("http://httpbin.org/post", data=dataDict) +Typically, you want to send some form-encoded data — much like an HTML form. +To do this, simply pass a dictionary to the `data` argument. Your dictionary of data will automatically be form-encoded when the request is made:: + + >>> payload = {'key1': 'value1', 'key2': 'value2'} + >>> r = requests.post("http://httpbin.org/post", data=payload) + >>> print r.content + { + "origin": "179.13.100.4", + "files": {}, + "form": { + "key2": "value2", + "key1": "value1" + }, + "url": "http://httpbin.org/post", + "args": {}, + "headers": { + "Content-Length": "23", + "Accept-Encoding": "identity, deflate, compress, gzip", + "Accept": "*/*", + "User-Agent": "python-requests/0.8.0", + "Host": "127.0.0.1:7077", + "Content-Type": "application/x-www-form-urlencoded" + }, + "data": "" + } + +There are many times that you want to send data that is not form-encoded. If you pass in a ``string`` instead of a ``dict``, that data will be posted directly. + +For example, the GitHub API v3 accepts JSON-Encoded POST/PATCH data:: + + url = 'https://api.github.com/some/endpoint' + payload = {'some': 'data'} + + r = requests.post(url, data=json.dumps(payload)) + + +Custom Headers +-------------- + +If you'd like to add HTTP headers to a request, simply pass in a ``dict`` to the +``headers`` parameter. + +For example, we didn't specify our content-type in the previous example:: + + url = 'https://api.github.com/some/endpoint' + payload = {'some': 'data'} + headers = {'content-type': 'application/json'} + + r = requests.post(url, data=json.dumps(payload), headers=headers) + + +POST a Multipart-Encoded File +----------------------------- + +Requests makes it simple to upload Multipart-encoded files:: + + >>> url = 'http://httpbin.org/post' + >>> files = {'report.xls': open('report.xls', 'rb')} + + >>> r = requests.post(url, data=None, files=files) >>> r.content - '{\n "origin": "::ffff:YourIpAddress", \n "files": {}, \n "form": {\n "key2": "value2", \n "key1": "value1"\n }, ... -Note the data= argument is equivalent to -d in cURL scripts. dataDict will -be form-encoded. + + + + Response Status Codes --------------------- @@ -73,7 +131,7 @@ reference:: >>> r.status_code == requests.codes.ok True -If we made a bad request, we can raise it with +If we made a bad request (non-200 response), we can raise it with :class:`Response.raise_for_status()`:: >>> _r = requests.get('http://httpbin.org/status/404') @@ -158,7 +216,7 @@ Basic Authentication Most web services require authentication. There many different types of authentication, but the most common is called HTTP Basic Auth. -Making requests with Basic Auth is easy, with Requests:: +Making requests with Basic Auth is extremely simple:: >>> requests.get('https://api.github.com/user', auth=('user', 'pass')) @@ -167,9 +225,7 @@ Making requests with Basic Auth is easy, with Requests:: Digest Authentication --------------------- -Another popular form of protecting web service is Digest Authentication. - -Requests supports it!:: +Another popular form of protecting web service is Digest Authentication:: >>> url = 'http://httpbin.org/digest-auth/auth/user/pass' >>> requests.get(url, auth=('digest', 'user', 'pass')) From df2c80e28eb2ed0dbc1a9f86ae65fb5e756936ce Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 12:45:35 -0800 Subject: [PATCH 227/255] fix form data tests --- test_requests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_requests.py b/test_requests.py index 40fa82e5..74c1b8b3 100755 --- a/test_requests.py +++ b/test_requests.py @@ -306,7 +306,7 @@ class RequestsTestSuite(unittest.TestCase): rbody = json.loads(r.content) # Body wasn't valid url encoded data, so the server returns None as # "form" and the raw body as "data". - self.assertEquals(rbody.get('form'), None) + self.assertAlmostEquals(rbody.get('form'), {}) self.assertEquals(rbody.get('data'), 'fooaowpeuf') @@ -354,7 +354,7 @@ class RequestsTestSuite(unittest.TestCase): rbody = json.loads(r.content) - self.assertEquals(rbody.get('form'), None) + self.assertEquals(rbody.get('form'), {}) self.assertEquals(rbody.get('data'), 'foobar') From 02408973c6da4bd89401d2a1994b4ea26dc260c9 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 12:47:04 -0800 Subject: [PATCH 228/255] keep-alive as config option only better defaults --- requests/defaults.py | 4 ++-- requests/sessions.py | 20 +++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/requests/defaults.py b/requests/defaults.py index eee4ae59..7a5a3fb8 100644 --- a/requests/defaults.py +++ b/requests/defaults.py @@ -35,8 +35,8 @@ defaults['base_headers'] = { defaults['verbose'] = None defaults['max_redirects'] = 30 defaults['decode_unicode'] = True -defaults['keep_alive'] = True defaults['pool_connections'] = 10 -defaults['pool_maxsize'] = 1 +defaults['pool_maxsize'] = 10 defaults['max_retries'] = 0 defaults['safe_mode'] = False +defaults['keep_alive'] = True diff --git a/requests/sessions.py b/requests/sessions.py index 6f7270bb..a1fb6e58 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -63,8 +63,7 @@ class Session(object): proxies=None, hooks=None, params=None, - config=None, - keep_alive=True): + config=None): self.headers = headers or {} self.cookies = cookies or {} @@ -74,7 +73,6 @@ class Session(object): self.hooks = hooks or {} self.params = params or {} self.config = config or {} - self.keep_alive = keep_alive for (k, v) in defaults.items(): self.config.setdefault(k, v) @@ -136,8 +134,12 @@ class Session(object): method = str(method).upper() - if cookies is None: - cookies = {} + # Default empty dicts for dict params. + cookies = {} if cookies is None else cookies + data = {} if data is None else data + files = {} if files is None else files + headers = {} if headers is None else headers + params = {} if params is None else params # Expand header values if headers: @@ -221,7 +223,7 @@ class Session(object): return self.request('HEAD', url, **kwargs) - def post(self, url, data='', **kwargs): + def post(self, url, data=None, **kwargs): """Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. @@ -232,7 +234,7 @@ class Session(object): return self.request('post', url, data=data, **kwargs) - def put(self, url, data='', **kwargs): + def put(self, url, data=None, **kwargs): """Sends a PUT request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. @@ -243,7 +245,7 @@ class Session(object): return self.request('put', url, data=data, **kwargs) - def patch(self, url, data='', **kwargs): + def patch(self, url, data=None, **kwargs): """Sends a PATCH request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. @@ -251,7 +253,7 @@ class Session(object): :param **kwargs: Optional arguments that ``request`` takes. """ - return self.request('patch', url, data='', **kwargs) + return self.request('patch', url, data=data, **kwargs) def delete(self, url, **kwargs): From 58e3dd9de85c98214b8838e133881fc263e5a3ac Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 12:47:21 -0800 Subject: [PATCH 229/255] fix urllib3 bug --- requests/packages/urllib3/connectionpool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 4c2c6b54..8b10dc70 100755 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -338,7 +338,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): except (SocketTimeout, Empty), e: # Timed out either by socket or queue - raise TimeoutError("Request timed out after %f seconds" % + raise TimeoutError("Request timed out after %s seconds" % self.timeout) except (BaseSSLError), e: From c0d8e8e8467a337f7ad0a10206b64087d8eef14a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 12:48:28 -0800 Subject: [PATCH 230/255] models cleanup --- requests/models.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index 64641c56..8bc2e3ad 100644 --- a/requests/models.py +++ b/requests/models.py @@ -23,6 +23,7 @@ from .packages.urllib3.exceptions import MaxRetryError from .packages.urllib3.exceptions import SSLError as _SSLError from .packages.urllib3.exceptions import HTTPError as _HTTPError from .packages.urllib3 import connectionpool, poolmanager +from .packages.urllib3.filepost import encode_multipart_formdata from .exceptions import ( Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError) from .utils import ( @@ -30,6 +31,8 @@ from .utils import ( stream_decode_response_unicode, decode_gzip, stream_decode_gzip) + + REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) @@ -330,8 +333,6 @@ class Request(object): body = None content_type = None - from .packages.urllib3.filepost import encode_multipart_formdata - # Multi-part file uploads. if self.files: if not isinstance(self.data, basestring): @@ -349,8 +350,12 @@ class Request(object): # TODO: Conflict? else: if self.data: + body = self._enc_data - content_type = 'application/x-www-form-urlencoded' + if isinstance(self.data, basestring): + content_type = None + else: + content_type = 'application/x-www-form-urlencoded' # Add content-type if it wasn't explicitly provided. if (content_type) and (not 'content-type' in self.headers): @@ -398,6 +403,7 @@ class Request(object): try: # Send the request. + r = conn.urlopen( method=self.method, url=url, From c05c64d21bd5144079b8dcfc85cf6ae5de115058 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 12:48:45 -0800 Subject: [PATCH 231/255] better defaults --- requests/api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requests/api.py b/requests/api.py index ff2e3b34..ee5fed62 100644 --- a/requests/api.py +++ b/requests/api.py @@ -90,7 +90,7 @@ def head(url, **kwargs): return request('HEAD', url, **kwargs) -def post(url, data='', **kwargs): +def post(url, data=None, **kwargs): """Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. @@ -101,7 +101,7 @@ def post(url, data='', **kwargs): return request('post', url, data=data, **kwargs) -def put(url, data='', **kwargs): +def put(url, data=None, **kwargs): """Sends a PUT request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. @@ -112,7 +112,7 @@ def put(url, data='', **kwargs): return request('put', url, data=data, **kwargs) -def patch(url, data='', **kwargs): +def patch(url, data=None, **kwargs): """Sends a PATCH request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. From f1e8700e9a6bdb8aaa3e4f86414b7ceeb729063c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 12:51:30 -0800 Subject: [PATCH 232/255] consistiency --- requests/api.py | 6 +++--- requests/sessions.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/requests/api.py b/requests/api.py index ee5fed62..fa41b8e8 100644 --- a/requests/api.py +++ b/requests/api.py @@ -65,7 +65,7 @@ def get(url, **kwargs): """ kwargs.setdefault('allow_redirects', True) - return request('GET', url, **kwargs) + return request('get', url, **kwargs) def options(url, **kwargs): @@ -76,7 +76,7 @@ def options(url, **kwargs): """ kwargs.setdefault('allow_redirects', True) - return request('OPTIONS', url, **kwargs) + return request('options', url, **kwargs) def head(url, **kwargs): @@ -87,7 +87,7 @@ def head(url, **kwargs): """ kwargs.setdefault('allow_redirects', True) - return request('HEAD', url, **kwargs) + return request('head', url, **kwargs) def post(url, data=None, **kwargs): diff --git a/requests/sessions.py b/requests/sessions.py index a1fb6e58..b59d1231 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -141,7 +141,7 @@ class Session(object): headers = {} if headers is None else headers params = {} if params is None else params - # Expand header values + # Expand header values. if headers: for k, v in headers.items() or {}: headers[k] = header_expand(v) @@ -198,7 +198,7 @@ class Session(object): """ kwargs.setdefault('allow_redirects', True) - return self.request('GET', url, **kwargs) + return self.request('get', url, **kwargs) def options(self, url, **kwargs): @@ -209,7 +209,7 @@ class Session(object): """ kwargs.setdefault('allow_redirects', True) - return self.request('OPTIONS', url, **kwargs) + return self.request('options', url, **kwargs) def head(self, url, **kwargs): @@ -220,7 +220,7 @@ class Session(object): """ kwargs.setdefault('allow_redirects', True) - return self.request('HEAD', url, **kwargs) + return self.request('head', url, **kwargs) def post(self, url, data=None, **kwargs): From 474e17b0af27b397e6abcb7eca3353775993426c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 12:53:23 -0800 Subject: [PATCH 233/255] remove bunk __all__s --- requests/api.py | 2 -- requests/async.py | 1 - 2 files changed, 3 deletions(-) diff --git a/requests/api.py b/requests/api.py index fa41b8e8..5f303521 100644 --- a/requests/api.py +++ b/requests/api.py @@ -13,8 +13,6 @@ This module implements the Requests API. from .sessions import session -__all__ = ('request', 'get', 'options', 'head', 'post', 'patch', 'put', 'delete') - def request(method, url, params=None, diff --git a/requests/async.py b/requests/async.py index 6b08f896..8bafb1ee 100644 --- a/requests/async.py +++ b/requests/async.py @@ -20,7 +20,6 @@ except ImportError: curious_george.patch_all(thread=False) from . import api -from .hooks import dispatch_hook __all__ = ( From 6a23625c1150cfc46c1a73d9c2472a2d2aa3fc60 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 12:57:22 -0800 Subject: [PATCH 234/255] give Request session reference for parameter+redirects --- requests/models.py | 6 ++++-- requests/sessions.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index 8bc2e3ad..1b8db94a 100644 --- a/requests/models.py +++ b/requests/models.py @@ -14,7 +14,6 @@ from Cookie import SimpleCookie from urlparse import urlparse, urlunparse, urljoin from datetime import datetime - from .auth import dispatch as auth_dispatch from .hooks import dispatch_hook from .structures import CaseInsensitiveDict @@ -117,6 +116,9 @@ class Request(object): #: Event-handling hooks. self.hooks = hooks + #: Session. + self.session = None + if headers: headers = CaseInsensitiveDict(self.headers) else: @@ -234,7 +236,7 @@ class Request(object): headers=headers, files=self.files, method=method, - # params=self.params, + params=self.session.params, auth=self._auth, cookies=cookies, redirect=True, diff --git a/requests/sessions.py b/requests/sessions.py index b59d1231..aec3f90f 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -175,6 +175,7 @@ class Session(object): # Create the (empty) response. r = Request(**args) + r.session = self # Don't send if asked nicely. if not return_response: From 3fccf63f491b53a443b6fb187da5db48ec33203d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 12:58:46 -0800 Subject: [PATCH 235/255] stylee --- requests/sessions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requests/sessions.py b/requests/sessions.py index aec3f90f..1ad3836c 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -163,18 +163,20 @@ class Session(object): _poolmanager=self.poolmanager ) + # Merge local kwargs with session kwargs. for attr in self.__attrs__: session_val = getattr(self, attr, None) local_val = args.get(attr) args[attr] = merge_kwargs(local_val, session_val) - # Arguments manipulation hook. args = dispatch_hook('args', args['hooks'], args) # Create the (empty) response. r = Request(**args) + + # Give the response some context. r.session = self # Don't send if asked nicely. From 40c54800a60f1c7e47f3de3e4a8482734bda1bdf Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 13:01:42 -0800 Subject: [PATCH 236/255] file uploads --- docs/user/quickstart.rst | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 7148fe6a..95add238 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -109,9 +109,26 @@ Requests makes it simple to upload Multipart-encoded files:: >>> url = 'http://httpbin.org/post' >>> files = {'report.xls': open('report.xls', 'rb')} - >>> r = requests.post(url, data=None, files=files) + >>> r = requests.post(url, files=files) >>> r.content - + { + "origin": "179.13.100.4", + "files": { + "hmm": "" + }, + "form": {}, + "url": "http://httpbin.org/post", + "args": {}, + "headers": { + "Content-Length": "3196", + "Accept-Encoding": "identity, deflate, compress, gzip", + "Accept": "*/*", + "User-Agent": "python-requests/0.8.0", + "Host": "httpbin.org:80", + "Content-Type": "multipart/form-data; boundary=127.0.0.1.502.21746.1321131593.786.1" + }, + "data": "" + } From 58c77bac27ce853d4b60a067d11a8b85dbc251b3 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 13:03:29 -0800 Subject: [PATCH 237/255] quicker start --- docs/user/quickstart.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 95add238..a8c8274b 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -132,8 +132,6 @@ Requests makes it simple to upload Multipart-encoded files:: - - Response Status Codes --------------------- @@ -231,7 +229,7 @@ Basic Authentication -------------------- Most web services require authentication. There many different types of -authentication, but the most common is called HTTP Basic Auth. +authentication, but the most common is HTTP Basic Auth. Making requests with Basic Auth is extremely simple:: @@ -242,7 +240,7 @@ Making requests with Basic Auth is extremely simple:: Digest Authentication --------------------- -Another popular form of protecting web service is Digest Authentication:: +Another popular form of web service protection is Digest Authentication:: >>> url = 'http://httpbin.org/digest-auth/auth/user/pass' >>> requests.get(url, auth=('digest', 'user', 'pass')) From 7ee2b7912707796e0abf0cd3201434198c800642 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 13:08:17 -0800 Subject: [PATCH 238/255] import async if it's available --- requests/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/requests/__init__.py b/requests/__init__.py index e3a9d2a0..b000557f 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -31,3 +31,8 @@ from .exceptions import ( RequestException, Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError ) + +try: + from . import async +except RuntimeError: + pass \ No newline at end of file From 6730818ddcfc5edcdf6aaebb17ca689f3ea39db6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 13:14:42 -0800 Subject: [PATCH 239/255] explicit --- requests/api.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/requests/api.py b/requests/api.py index 5f303521..56702db1 100644 --- a/requests/api.py +++ b/requests/api.py @@ -48,9 +48,21 @@ def request(method, url, s = session() return s.request( - method, url, params, data, headers, cookies, files, auth, - timeout, allow_redirects, proxies, hooks, return_response, - config, prefetch + method=method, + url=url, + params=params, + data=data, + headers=headers, + cookies=cookies, + files=files, + auth=auth, + timeout=timeout, + allow_redirects=allow_redirects, + proxies=proxies, + hooks=hooks, + return_response=return_response, + config=config, + prefetch=prefetch ) From cfc1a76381ac3b728456f0d02a81acf38cfca4db Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 13:14:51 -0800 Subject: [PATCH 240/255] no auto async --- requests/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/requests/__init__.py b/requests/__init__.py index b000557f..e3a9d2a0 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -31,8 +31,3 @@ from .exceptions import ( RequestException, Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError ) - -try: - from . import async -except RuntimeError: - pass \ No newline at end of file From e84d84b5b8660efa66feb66564683628944505bb Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 16:39:18 -0500 Subject: [PATCH 241/255] use new collections MutableMutex --- requests/packages/urllib3/_collections.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py index e73f0ed4..912c8666 100755 --- a/requests/packages/urllib3/_collections.py +++ b/requests/packages/urllib3/_collections.py @@ -4,10 +4,12 @@ # This module is part of urllib3 and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php -from collections import deque, MutableMapping +from collections import deque from threading import RLock +from .__collections import MutableMapping + __all__ = ['RecentlyUsedContainer'] From ec926a49580619a03c36f58f02df0874fbf9303b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 16:46:48 -0500 Subject: [PATCH 242/255] test fix --- requests/sessions.py | 1 - requests/structures.py | 1 + test_requests.py | 3 +-- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index 1ad3836c..9610fd5e 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -269,7 +269,6 @@ class Session(object): return self.request('delete', url, **kwargs) - def session(**kwargs): """Returns a :class:`Session` for context-management.""" diff --git a/requests/structures.py b/requests/structures.py index f23f0b38..35a903fd 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -8,6 +8,7 @@ Data structures that power Requests. """ + class CaseInsensitiveDict(dict): """Case-insensitive Dictionary diff --git a/test_requests.py b/test_requests.py index 74c1b8b3..61953a37 100755 --- a/test_requests.py +++ b/test_requests.py @@ -306,7 +306,7 @@ class RequestsTestSuite(unittest.TestCase): rbody = json.loads(r.content) # Body wasn't valid url encoded data, so the server returns None as # "form" and the raw body as "data". - self.assertAlmostEquals(rbody.get('form'), {}) + self.assertEquals(rbody.get('form'), {}) self.assertEquals(rbody.get('data'), 'fooaowpeuf') @@ -538,7 +538,6 @@ class RequestsTestSuite(unittest.TestCase): assert params3['c'] in r3.content def test_invalid_content(self): - # WARNING: if you're using a terrible DNS provider (comcast), # this will fail. try: From 0a4b7356ac039ffc0eed3086ced33d5f6b54dbba Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 16:47:14 -0500 Subject: [PATCH 243/255] rip 2.7.x collections out --- requests/packages/urllib3/__collections.py | 991 +++++++++++++++++++++ 1 file changed, 991 insertions(+) create mode 100644 requests/packages/urllib3/__collections.py diff --git a/requests/packages/urllib3/__collections.py b/requests/packages/urllib3/__collections.py new file mode 100644 index 00000000..514d0027 --- /dev/null +++ b/requests/packages/urllib3/__collections.py @@ -0,0 +1,991 @@ +from __future__ import with_statement + + +# Copyright 2007 Google, Inc. All Rights Reserved. +# Licensed to PSF under a Contributor Agreement. + +"""Abstract Base Classes (ABCs) according to PEP 3119.""" + +import types + +from _weakref import ref + +__all__ = ['WeakSet'] + + +class _IterationGuard(object): + # This context manager registers itself in the current iterators of the + # weak container, such as to delay all removals until the context manager + # exits. + # This technique should be relatively thread-safe (since sets are). + + def __init__(self, weakcontainer): + # Don't create cycles + self.weakcontainer = ref(weakcontainer) + + def __enter__(self): + w = self.weakcontainer() + if w is not None: + w._iterating.add(self) + return self + + def __exit__(self, e, t, b): + w = self.weakcontainer() + if w is not None: + s = w._iterating + s.remove(self) + if not s: + w._commit_removals() + + +class WeakSet(object): + def __init__(self, data=None): + self.data = set() + def _remove(item, selfref=ref(self)): + self = selfref() + if self is not None: + if self._iterating: + self._pending_removals.append(item) + else: + self.data.discard(item) + self._remove = _remove + # A list of keys to be removed + self._pending_removals = [] + self._iterating = set() + if data is not None: + self.update(data) + + def _commit_removals(self): + l = self._pending_removals + discard = self.data.discard + while l: + discard(l.pop()) + + def __iter__(self): + with _IterationGuard(self): + for itemref in self.data: + item = itemref() + if item is not None: + yield item + + def __len__(self): + return sum(x() is not None for x in self.data) + + def __contains__(self, item): + try: + wr = ref(item) + except TypeError: + return False + return wr in self.data + + def __reduce__(self): + return (self.__class__, (list(self),), + getattr(self, '__dict__', None)) + + __hash__ = None + + def add(self, item): + if self._pending_removals: + self._commit_removals() + self.data.add(ref(item, self._remove)) + + def clear(self): + if self._pending_removals: + self._commit_removals() + self.data.clear() + + def copy(self): + return self.__class__(self) + + def pop(self): + if self._pending_removals: + self._commit_removals() + while True: + try: + itemref = self.data.pop() + except KeyError: + raise KeyError('pop from empty WeakSet') + item = itemref() + if item is not None: + return item + + def remove(self, item): + if self._pending_removals: + self._commit_removals() + self.data.remove(ref(item)) + + def discard(self, item): + if self._pending_removals: + self._commit_removals() + self.data.discard(ref(item)) + + def update(self, other): + if self._pending_removals: + self._commit_removals() + if isinstance(other, self.__class__): + self.data.update(other.data) + else: + for element in other: + self.add(element) + + def __ior__(self, other): + self.update(other) + return self + + # Helper functions for simple delegating methods. + def _apply(self, other, method): + if not isinstance(other, self.__class__): + other = self.__class__(other) + newdata = method(other.data) + newset = self.__class__() + newset.data = newdata + return newset + + def difference(self, other): + return self._apply(other, self.data.difference) + __sub__ = difference + + def difference_update(self, other): + if self._pending_removals: + self._commit_removals() + if self is other: + self.data.clear() + else: + self.data.difference_update(ref(item) for item in other) + def __isub__(self, other): + if self._pending_removals: + self._commit_removals() + if self is other: + self.data.clear() + else: + self.data.difference_update(ref(item) for item in other) + return self + + def intersection(self, other): + return self._apply(other, self.data.intersection) + __and__ = intersection + + def intersection_update(self, other): + if self._pending_removals: + self._commit_removals() + self.data.intersection_update(ref(item) for item in other) + def __iand__(self, other): + if self._pending_removals: + self._commit_removals() + self.data.intersection_update(ref(item) for item in other) + return self + + def issubset(self, other): + return self.data.issubset(ref(item) for item in other) + __lt__ = issubset + + def __le__(self, other): + return self.data <= set(ref(item) for item in other) + + def issuperset(self, other): + return self.data.issuperset(ref(item) for item in other) + __gt__ = issuperset + + def __ge__(self, other): + return self.data >= set(ref(item) for item in other) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return NotImplemented + return self.data == set(ref(item) for item in other) + + def symmetric_difference(self, other): + return self._apply(other, self.data.symmetric_difference) + __xor__ = symmetric_difference + + def symmetric_difference_update(self, other): + if self._pending_removals: + self._commit_removals() + if self is other: + self.data.clear() + else: + self.data.symmetric_difference_update(ref(item) for item in other) + def __ixor__(self, other): + if self._pending_removals: + self._commit_removals() + if self is other: + self.data.clear() + else: + self.data.symmetric_difference_update(ref(item) for item in other) + return self + + def union(self, other): + return self._apply(other, self.data.union) + __or__ = union + + def isdisjoint(self, other): + return len(self.intersection(other)) == 0 + + +# Instance of old-style class +class _C: pass +_InstanceType = type(_C()) + + +def abstractmethod(funcobj): + """A decorator indicating abstract methods. + + Requires that the metaclass is ABCMeta or derived from it. A + class that has a metaclass derived from ABCMeta cannot be + instantiated unless all of its abstract methods are overridden. + The abstract methods can be called using any of the normal + 'super' call mechanisms. + + Usage: + + class C: + __metaclass__ = ABCMeta + @abstractmethod + def my_abstract_method(self, ...): + ... + """ + funcobj.__isabstractmethod__ = True + return funcobj + + +class abstractproperty(property): + """A decorator indicating abstract properties. + + Requires that the metaclass is ABCMeta or derived from it. A + class that has a metaclass derived from ABCMeta cannot be + instantiated unless all of its abstract properties are overridden. + The abstract properties can be called using any of the normal + 'super' call mechanisms. + + Usage: + + class C: + __metaclass__ = ABCMeta + @abstractproperty + def my_abstract_property(self): + ... + + This defines a read-only property; you can also define a read-write + abstract property using the 'long' form of property declaration: + + class C: + __metaclass__ = ABCMeta + def getx(self): ... + def setx(self, value): ... + x = abstractproperty(getx, setx) + """ + __isabstractmethod__ = True + + +class ABCMeta(type): + + """Metaclass for defining Abstract Base Classes (ABCs). + + Use this metaclass to create an ABC. An ABC can be subclassed + directly, and then acts as a mix-in class. You can also register + unrelated concrete classes (even built-in classes) and unrelated + ABCs as 'virtual subclasses' -- these and their descendants will + be considered subclasses of the registering ABC by the built-in + issubclass() function, but the registering ABC won't show up in + their MRO (Method Resolution Order) nor will method + implementations defined by the registering ABC be callable (not + even via super()). + + """ + + # A global counter that is incremented each time a class is + # registered as a virtual subclass of anything. It forces the + # negative cache to be cleared before its next use. + _abc_invalidation_counter = 0 + + def __new__(mcls, name, bases, namespace): + cls = super(ABCMeta, mcls).__new__(mcls, name, bases, namespace) + # Compute set of abstract method names + abstracts = set(name + for name, value in namespace.items() + if getattr(value, "__isabstractmethod__", False)) + for base in bases: + for name in getattr(base, "__abstractmethods__", set()): + value = getattr(cls, name, None) + if getattr(value, "__isabstractmethod__", False): + abstracts.add(name) + cls.__abstractmethods__ = frozenset(abstracts) + # Set up inheritance registry + cls._abc_registry = WeakSet() + cls._abc_cache = WeakSet() + cls._abc_negative_cache = WeakSet() + cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + return cls + + def register(cls, subclass): + """Register a virtual subclass of an ABC.""" + if not isinstance(subclass, (type, types.ClassType)): + raise TypeError("Can only register classes") + if issubclass(subclass, cls): + return # Already a subclass + # Subtle: test for cycles *after* testing for "already a subclass"; + # this means we allow X.register(X) and interpret it as a no-op. + if issubclass(cls, subclass): + # This would create a cycle, which is bad for the algorithm below + raise RuntimeError("Refusing to create an inheritance cycle") + cls._abc_registry.add(subclass) + ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache + + def _dump_registry(cls, file=None): + """Debug helper to print the ABC registry.""" + print >> file, "Class: %s.%s" % (cls.__module__, cls.__name__) + print >> file, "Inv.counter: %s" % ABCMeta._abc_invalidation_counter + for name in sorted(cls.__dict__.keys()): + if name.startswith("_abc_"): + value = getattr(cls, name) + print >> file, "%s: %r" % (name, value) + + def __instancecheck__(cls, instance): + """Override for isinstance(instance, cls).""" + # Inline the cache checking when it's simple. + subclass = getattr(instance, '__class__', None) + if subclass is not None and subclass in cls._abc_cache: + return True + subtype = type(instance) + # Old-style instances + if subtype is _InstanceType: + subtype = subclass + if subtype is subclass or subclass is None: + if (cls._abc_negative_cache_version == + ABCMeta._abc_invalidation_counter and + subtype in cls._abc_negative_cache): + return False + # Fall back to the subclass check. + return cls.__subclasscheck__(subtype) + return (cls.__subclasscheck__(subclass) or + cls.__subclasscheck__(subtype)) + + def __subclasscheck__(cls, subclass): + """Override for issubclass(subclass, cls).""" + # Check cache + if subclass in cls._abc_cache: + return True + # Check negative cache; may have to invalidate + if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter: + # Invalidate the negative cache + cls._abc_negative_cache = WeakSet() + cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter + elif subclass in cls._abc_negative_cache: + return False + # Check the subclass hook + ok = cls.__subclasshook__(subclass) + if ok is not NotImplemented: + assert isinstance(ok, bool) + if ok: + cls._abc_cache.add(subclass) + else: + cls._abc_negative_cache.add(subclass) + return ok + # Check if it's a direct subclass + if cls in getattr(subclass, '__mro__', ()): + cls._abc_cache.add(subclass) + return True + # Check if it's a subclass of a registered class (recursive) + for rcls in cls._abc_registry: + if issubclass(subclass, rcls): + cls._abc_cache.add(subclass) + return True + # Check if it's a subclass of a subclass (recursive) + for scls in cls.__subclasses__(): + if issubclass(subclass, scls): + cls._abc_cache.add(subclass) + return True + # No dice; update negative cache + cls._abc_negative_cache.add(subclass) + return False + + + + + + + + +import sys + +### ONE-TRICK PONIES ### + +def _hasattr(C, attr): + try: + return any(attr in B.__dict__ for B in C.__mro__) + except AttributeError: + # Old-style class + return hasattr(C, attr) + + +class Hashable: + __metaclass__ = ABCMeta + + @abstractmethod + def __hash__(self): + return 0 + + @classmethod + def __subclasshook__(cls, C): + if cls is Hashable: + try: + for B in C.__mro__: + if "__hash__" in B.__dict__: + if B.__dict__["__hash__"]: + return True + break + except AttributeError: + # Old-style class + if getattr(C, "__hash__", None): + return True + return NotImplemented + + +class Iterable: + __metaclass__ = ABCMeta + + @abstractmethod + def __iter__(self): + while False: + yield None + + @classmethod + def __subclasshook__(cls, C): + if cls is Iterable: + if _hasattr(C, "__iter__"): + return True + return NotImplemented + +Iterable.register(str) + + +class Iterator(Iterable): + + @abstractmethod + def next(self): + raise StopIteration + + def __iter__(self): + return self + + @classmethod + def __subclasshook__(cls, C): + if cls is Iterator: + if _hasattr(C, "next") and _hasattr(C, "__iter__"): + return True + return NotImplemented + + +class Sized: + __metaclass__ = ABCMeta + + @abstractmethod + def __len__(self): + return 0 + + @classmethod + def __subclasshook__(cls, C): + if cls is Sized: + if _hasattr(C, "__len__"): + return True + return NotImplemented + + +class Container: + __metaclass__ = ABCMeta + + @abstractmethod + def __contains__(self, x): + return False + + @classmethod + def __subclasshook__(cls, C): + if cls is Container: + if _hasattr(C, "__contains__"): + return True + return NotImplemented + + +class Callable: + __metaclass__ = ABCMeta + + @abstractmethod + def __call__(self, *args, **kwds): + return False + + @classmethod + def __subclasshook__(cls, C): + if cls is Callable: + if _hasattr(C, "__call__"): + return True + return NotImplemented + + +### SETS ### + + +class Set(Sized, Iterable, Container): + """A set is a finite, iterable container. + + This class provides concrete generic implementations of all + methods except for __contains__, __iter__ and __len__. + + To override the comparisons (presumably for speed, as the + semantics are fixed), all you have to do is redefine __le__ and + then the other operations will automatically follow suit. + """ + + def __le__(self, other): + if not isinstance(other, Set): + return NotImplemented + if len(self) > len(other): + return False + for elem in self: + if elem not in other: + return False + return True + + def __lt__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) < len(other) and self.__le__(other) + + def __gt__(self, other): + if not isinstance(other, Set): + return NotImplemented + return other < self + + def __ge__(self, other): + if not isinstance(other, Set): + return NotImplemented + return other <= self + + def __eq__(self, other): + if not isinstance(other, Set): + return NotImplemented + return len(self) == len(other) and self.__le__(other) + + def __ne__(self, other): + return not (self == other) + + @classmethod + def _from_iterable(cls, it): + '''Construct an instance of the class from any iterable input. + + Must override this method if the class constructor signature + does not accept an iterable for an input. + ''' + return cls(it) + + def __and__(self, other): + if not isinstance(other, Iterable): + return NotImplemented + return self._from_iterable(value for value in other if value in self) + + def isdisjoint(self, other): + for value in other: + if value in self: + return False + return True + + def __or__(self, other): + if not isinstance(other, Iterable): + return NotImplemented + chain = (e for s in (self, other) for e in s) + return self._from_iterable(chain) + + def __sub__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return self._from_iterable(value for value in self + if value not in other) + + def __xor__(self, other): + if not isinstance(other, Set): + if not isinstance(other, Iterable): + return NotImplemented + other = self._from_iterable(other) + return (self - other) | (other - self) + + # Sets are not hashable by default, but subclasses can change this + __hash__ = None + + def _hash(self): + """Compute the hash value of a set. + + Note that we don't define __hash__: not all sets are hashable. + But if you define a hashable set type, its __hash__ should + call this function. + + This must be compatible __eq__. + + All sets ought to compare equal if they contain the same + elements, regardless of how they are implemented, and + regardless of the order of the elements; so there's not much + freedom for __eq__ or __hash__. We match the algorithm used + by the built-in frozenset type. + """ + MAX = sys.maxint + MASK = 2 * MAX + 1 + n = len(self) + h = 1927868237 * (n + 1) + h &= MASK + for x in self: + hx = hash(x) + h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167 + h &= MASK + h = h * 69069 + 907133923 + h &= MASK + if h > MAX: + h -= MASK + 1 + if h == -1: + h = 590923713 + return h + +Set.register(frozenset) + + +class MutableSet(Set): + + @abstractmethod + def add(self, value): + """Add an element.""" + raise NotImplementedError + + @abstractmethod + def discard(self, value): + """Remove an element. Do not raise an exception if absent.""" + raise NotImplementedError + + def remove(self, value): + """Remove an element. If not a member, raise a KeyError.""" + if value not in self: + raise KeyError(value) + self.discard(value) + + def pop(self): + """Return the popped value. Raise KeyError if empty.""" + it = iter(self) + try: + value = next(it) + except StopIteration: + raise KeyError + self.discard(value) + return value + + def clear(self): + """This is slow (creates N new iterators!) but effective.""" + try: + while True: + self.pop() + except KeyError: + pass + + def __ior__(self, it): + for value in it: + self.add(value) + return self + + def __iand__(self, it): + for value in (self - it): + self.discard(value) + return self + + def __ixor__(self, it): + if it is self: + self.clear() + else: + if not isinstance(it, Set): + it = self._from_iterable(it) + for value in it: + if value in self: + self.discard(value) + else: + self.add(value) + return self + + def __isub__(self, it): + if it is self: + self.clear() + else: + for value in it: + self.discard(value) + return self + +MutableSet.register(set) + + +### MAPPINGS ### + + +class Mapping(Sized, Iterable, Container): + + @abstractmethod + def __getitem__(self, key): + raise KeyError + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + def __contains__(self, key): + try: + self[key] + except KeyError: + return False + else: + return True + + def iterkeys(self): + return iter(self) + + def itervalues(self): + for key in self: + yield self[key] + + def iteritems(self): + for key in self: + yield (key, self[key]) + + def keys(self): + return list(self) + + def items(self): + return [(key, self[key]) for key in self] + + def values(self): + return [self[key] for key in self] + + # Mappings are not hashable by default, but subclasses can change this + __hash__ = None + + def __eq__(self, other): + if not isinstance(other, Mapping): + return NotImplemented + return dict(self.items()) == dict(other.items()) + + def __ne__(self, other): + return not (self == other) + +class MappingView(Sized): + + def __init__(self, mapping): + self._mapping = mapping + + def __len__(self): + return len(self._mapping) + + def __repr__(self): + return '{0.__class__.__name__}({0._mapping!r})'.format(self) + + +class KeysView(MappingView, Set): + + @classmethod + def _from_iterable(self, it): + return set(it) + + def __contains__(self, key): + return key in self._mapping + + def __iter__(self): + for key in self._mapping: + yield key + + +class ItemsView(MappingView, Set): + + @classmethod + def _from_iterable(self, it): + return set(it) + + def __contains__(self, item): + key, value = item + try: + v = self._mapping[key] + except KeyError: + return False + else: + return v == value + + def __iter__(self): + for key in self._mapping: + yield (key, self._mapping[key]) + + +class ValuesView(MappingView): + + def __contains__(self, value): + for key in self._mapping: + if value == self._mapping[key]: + return True + return False + + def __iter__(self): + for key in self._mapping: + yield self._mapping[key] + + +class MutableMapping(Mapping): + + @abstractmethod + def __setitem__(self, key, value): + raise KeyError + + @abstractmethod + def __delitem__(self, key): + raise KeyError + + __marker = object() + + def pop(self, key, default=__marker): + try: + value = self[key] + except KeyError: + if default is self.__marker: + raise + return default + else: + del self[key] + return value + + def popitem(self): + try: + key = next(iter(self)) + except StopIteration: + raise KeyError + value = self[key] + del self[key] + return key, value + + def clear(self): + try: + while True: + self.popitem() + except KeyError: + pass + + def update(*args, **kwds): + if len(args) > 2: + raise TypeError("update() takes at most 2 positional " + "arguments ({} given)".format(len(args))) + elif not args: + raise TypeError("update() takes at least 1 argument (0 given)") + self = args[0] + other = args[1] if len(args) >= 2 else () + + if isinstance(other, Mapping): + for key in other: + self[key] = other[key] + elif hasattr(other, "keys"): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwds.items(): + self[key] = value + + def setdefault(self, key, default=None): + try: + return self[key] + except KeyError: + self[key] = default + return default + +MutableMapping.register(dict) + + +### SEQUENCES ### + + +class Sequence(Sized, Iterable, Container): + """All the operations on a read-only sequence. + + Concrete subclasses must override __new__ or __init__, + __getitem__, and __len__. + """ + + @abstractmethod + def __getitem__(self, index): + raise IndexError + + def __iter__(self): + i = 0 + try: + while True: + v = self[i] + yield v + i += 1 + except IndexError: + return + + def __contains__(self, value): + for v in self: + if v == value: + return True + return False + + def __reversed__(self): + for i in reversed(range(len(self))): + yield self[i] + + def index(self, value): + for i, v in enumerate(self): + if v == value: + return i + raise ValueError + + def count(self, value): + return sum(1 for v in self if v == value) + +Sequence.register(tuple) +Sequence.register(basestring) +Sequence.register(buffer) +Sequence.register(xrange) + + +class MutableSequence(Sequence): + + @abstractmethod + def __setitem__(self, index, value): + raise IndexError + + @abstractmethod + def __delitem__(self, index): + raise IndexError + + @abstractmethod + def insert(self, index, value): + raise IndexError + + def append(self, value): + self.insert(len(self), value) + + def reverse(self): + n = len(self) + for i in range(n//2): + self[i], self[n-i-1] = self[n-i-1], self[i] + + def extend(self, values): + for v in values: + self.append(v) + + def pop(self, index=-1): + v = self[index] + del self[index] + return v + + def remove(self, value): + del self[self.index(value)] + + def __iadd__(self, values): + self.extend(values) + return self + +MutableSequence.register(list) + From 2584c48e8d744148e96a4ed5d8feda824887398d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 16:47:40 -0500 Subject: [PATCH 244/255] python2.5 cookie support --- requests/models.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/requests/models.py b/requests/models.py index 1b8db94a..e70d9db8 100644 --- a/requests/models.py +++ b/requests/models.py @@ -5,12 +5,13 @@ requests.models ~~~~~~~~~~~~~~~ +This module contains the primary objects that power Requests. """ import urllib import zlib -from Cookie import SimpleCookie +from Cookie import SimpleCookie from urlparse import urlparse, urlunparse, urljoin from datetime import datetime @@ -26,8 +27,8 @@ from .packages.urllib3.filepost import encode_multipart_formdata from .exceptions import ( Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError) from .utils import ( - dict_from_cookiejar, get_unicode_from_response, - stream_decode_response_unicode, decode_gzip, stream_decode_gzip) + get_unicode_from_response, stream_decode_response_unicode, + decode_gzip, stream_decode_gzip) @@ -393,9 +394,9 @@ class Request(object): if 'cookie' not in self.headers: # Simple cookie with our dict. - # TODO: Multi-value headers. c = SimpleCookie() - c.load(self.cookies) + for (k, v) in self.cookies.items(): + c[k] = v # Turn it into a header. cookie_header = c.output(header='').strip() From 3462d9c87b92bd6813006ee8eb31d7b8f7b4335e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 16:49:08 -0500 Subject: [PATCH 245/255] trust built-in if its available --- requests/packages/urllib3/_collections.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py index 912c8666..ccc39734 100755 --- a/requests/packages/urllib3/_collections.py +++ b/requests/packages/urllib3/_collections.py @@ -8,7 +8,10 @@ from collections import deque from threading import RLock -from .__collections import MutableMapping +try: + from collections import MutableMapping +except ImportError: + from .__collections import MutableMapping __all__ = ['RecentlyUsedContainer'] From 0a24de2ef9f334ed86512434bd86d4e886e82f97 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 18:23:19 -0500 Subject: [PATCH 246/255] big docs update on workflow of request lifecycle --- docs/user/advanced.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index d4a416cf..70514ef7 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -46,6 +46,27 @@ Any dictionaries that you pass to a request method will be merged with the sessi All values that are contained within a session are directly available to you. See the :ref:`Session API Docs ` to learn more. +Body Content Workflow +---------------------- + +By default, When you make a request, the body of the response isn't downloaded immediately. The response headers are downloaded when you make a request, but the content isn't downloaded until you access the :class:`Response.content` attribute. + +Let's walk through it:: + + tarball_url = 'https://github.com/kennethreitz/requests/tarball/master' + r = requests.get(tarball_url) + +The request has been made, but the connection is still open. The response body has not been downloaded yet. :: + + r.content + +The content has been downloaded and cached. + +You can override this default behavior with the ``prefetch`` parameter:: + + r = requests.get(tarball_url, prefetch=True) + # Blocks until all of request body has been downloaded. + Configuring Requests -------------------- From 30377ca91211725aacf41905c5998cd4ed36a900 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 18:23:26 -0500 Subject: [PATCH 247/255] redirection docs --- docs/user/quickstart.rst | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index a8c8274b..b77bd294 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -247,6 +247,43 @@ Another popular form of web service protection is Digest Authentication:: +Redirection and History +----------------------- + +Requests will automatically perform location redirection while using impodotent methods. + +GitHub redirects all HTTP requests to HTTPS. Let's see what happens:: + + >>> r = request.get('http://github.com') + >>> r.url + 'https://github.com/' + >>> r.status_code + 200 + >>> r.history + [] + +The :class:`Response.history` list contains a list of the +:class:`Request` objects that were created in order to complete the request. + +If you're using GET, HEAD, or OPTIONS, you can disable redirection +handling with the ``disable_redirects`` parameter:: + + >>> r = request.get('http://github.com') + >>> r.status_code + 301 + >>> r.history + [] + +If you're using POST, PUT, PATCH, *&c*, you can also explicitly enable redirection as well:: + + >>> r = request.post('http://github.com') + >>> r.url + 'https://github.com/' + >>> r.history + [] + + + ----------------------- Ready for more? Check out the :ref:`advanced ` section. From 4ab76943efb509032fb7d599c47b60cd6f8a6a1f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 18:34:27 -0500 Subject: [PATCH 248/255] document timeouts --- docs/user/quickstart.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index b77bd294..4ead8755 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -283,6 +283,38 @@ If you're using POST, PUT, PATCH, *&c*, you can also explicitly enable redirecti [] +Timeouts +-------- + +You can tell requests to stop waiting for a response after a given number of seconds with the ``timeout`` parameter:: + + >>> requests.get('http://github.com', timeout=0.001) + Traceback (most recent call last): + File "", line 1, in + requests.exceptions.Timeout: Request timed out. + +.. admonition:: Note + + ``timeout`` only effects the connection process itself, not the downloading of the respone body. + + +Note + +Errors and Exceptions +--------------------- + +In the event of a network problem (e.g. DNS failure, refused connection, etc), +Requests will raise a :class:`ConnectionError` exception. + +In the event of the rare invalid HTTP response, Requests will raise +an :class:`HTTPError` exception. + +If a request times out, a :class:`Timeout` exception is raised. + +If a request exceeds the configured number of maximum redirections, a :class:`TooManyRedirects` exception is raised. + +All exceptions that Requests explicitly raises inherit from +:class:`requests.exceptions.RequestException`. ----------------------- From 37bbdbe8c3d041e5bb66396f735266c4dfeb5652 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 18:34:33 -0500 Subject: [PATCH 249/255] urllib3 in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6b95c31e..17e612ba 100755 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ setup( packages= [ 'requests', 'requests.packages', - 'requests.packages.poster' + 'requests.packages.urllib3' ], install_requires=required, license='ISC', From 14a798882c75323848c342f0aa53dc7d8c8751aa Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 12 Nov 2011 18:34:40 -0500 Subject: [PATCH 250/255] fixes --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index df7ab830..bab640b0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -33,7 +33,7 @@ See `the same code, without Requests `_. Requests allow you to send **HEAD**, **GET**, **POST**, **PUT**, **PATCH**, and **DELETE** HTTP requests. You can add headers, form data, multipart files, and parameters with simple Python dictionaries, and access the -response data in the same way. It's powered by :py:class:`httplib` and urllib3, and it works just as you'd expect. +response data in the same way. It's powered by :py:class:`httplib` and :py:class:`urllib3`, and it strives to be as elegant and approachable as possible. Testimonials ------------ From e8668fb5129c27f38d2bc4917e7f9abfe946f9ff Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sat, 12 Nov 2011 18:16:20 -0800 Subject: [PATCH 251/255] Adding me into a Urllib3 subsection (as was in urllib3-backup). --- AUTHORS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/AUTHORS b/AUTHORS index 20756691..f0b66eda 100644 --- a/AUTHORS +++ b/AUTHORS @@ -7,6 +7,12 @@ Development Lead - Kenneth Reitz +Urllib3 +``````` + +- Andrey Petrov + + Patches and Suggestions ``````````````````````` From 0086f97b6553ac323565e8653eae678e380219ae Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 13 Nov 2011 00:36:22 -0500 Subject: [PATCH 252/255] Guess filename. Closes #192 --- requests/models.py | 5 +++-- requests/utils.py | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index e70d9db8..79a15fc2 100644 --- a/requests/models.py +++ b/requests/models.py @@ -28,7 +28,7 @@ from .exceptions import ( Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError) from .utils import ( get_unicode_from_response, stream_decode_response_unicode, - decode_gzip, stream_decode_gzip) + decode_gzip, stream_decode_gzip, guess_filename) @@ -346,7 +346,8 @@ class Request(object): fields = dict(self.data) for (k, v) in self.files.items(): - fields.update({k: (k, v.read())}) + fields.update({k: (guess_filename(k) or k, v.read())}) + (body, content_type) = encode_multipart_formdata(fields) else: pass diff --git a/requests/utils.py b/requests/utils.py index bd7dead4..0249e9d6 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -20,6 +20,12 @@ import zlib from urllib2 import parse_http_list as _parse_list_header +def guess_filename(obj): + """Tries to guess the filename of the given object.""" + name = getattr(obj, 'name', None) + if name and name[0] != '<' and name[-1] != '>': + return name + # From mitsuhiko/werkzeug (used with permission). def parse_list_header(value): """Parse lists as described by RFC 2068 Section 2. From 3458e6a905b543f499aea95e463bb653114ffb1c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 13 Nov 2011 00:38:08 -0500 Subject: [PATCH 253/255] upstream urllib3 --- requests/packages/urllib3/__collections.py | 991 ------------------ requests/packages/urllib3/__init__.py | 0 requests/packages/urllib3/_collections.py | 34 +- requests/packages/urllib3/connectionpool.py | 2 +- requests/packages/urllib3/contrib/__init__.py | 0 requests/packages/urllib3/contrib/ntlmpool.py | 0 requests/packages/urllib3/exceptions.py | 0 requests/packages/urllib3/filepost.py | 0 requests/packages/urllib3/poolmanager.py | 0 requests/packages/urllib3/request.py | 0 requests/packages/urllib3/response.py | 0 11 files changed, 14 insertions(+), 1013 deletions(-) delete mode 100644 requests/packages/urllib3/__collections.py mode change 100755 => 100644 requests/packages/urllib3/__init__.py mode change 100755 => 100644 requests/packages/urllib3/_collections.py mode change 100755 => 100644 requests/packages/urllib3/connectionpool.py mode change 100755 => 100644 requests/packages/urllib3/contrib/__init__.py mode change 100755 => 100644 requests/packages/urllib3/contrib/ntlmpool.py mode change 100755 => 100644 requests/packages/urllib3/exceptions.py mode change 100755 => 100644 requests/packages/urllib3/filepost.py mode change 100755 => 100644 requests/packages/urllib3/poolmanager.py mode change 100755 => 100644 requests/packages/urllib3/request.py mode change 100755 => 100644 requests/packages/urllib3/response.py diff --git a/requests/packages/urllib3/__collections.py b/requests/packages/urllib3/__collections.py deleted file mode 100644 index 514d0027..00000000 --- a/requests/packages/urllib3/__collections.py +++ /dev/null @@ -1,991 +0,0 @@ -from __future__ import with_statement - - -# Copyright 2007 Google, Inc. All Rights Reserved. -# Licensed to PSF under a Contributor Agreement. - -"""Abstract Base Classes (ABCs) according to PEP 3119.""" - -import types - -from _weakref import ref - -__all__ = ['WeakSet'] - - -class _IterationGuard(object): - # This context manager registers itself in the current iterators of the - # weak container, such as to delay all removals until the context manager - # exits. - # This technique should be relatively thread-safe (since sets are). - - def __init__(self, weakcontainer): - # Don't create cycles - self.weakcontainer = ref(weakcontainer) - - def __enter__(self): - w = self.weakcontainer() - if w is not None: - w._iterating.add(self) - return self - - def __exit__(self, e, t, b): - w = self.weakcontainer() - if w is not None: - s = w._iterating - s.remove(self) - if not s: - w._commit_removals() - - -class WeakSet(object): - def __init__(self, data=None): - self.data = set() - def _remove(item, selfref=ref(self)): - self = selfref() - if self is not None: - if self._iterating: - self._pending_removals.append(item) - else: - self.data.discard(item) - self._remove = _remove - # A list of keys to be removed - self._pending_removals = [] - self._iterating = set() - if data is not None: - self.update(data) - - def _commit_removals(self): - l = self._pending_removals - discard = self.data.discard - while l: - discard(l.pop()) - - def __iter__(self): - with _IterationGuard(self): - for itemref in self.data: - item = itemref() - if item is not None: - yield item - - def __len__(self): - return sum(x() is not None for x in self.data) - - def __contains__(self, item): - try: - wr = ref(item) - except TypeError: - return False - return wr in self.data - - def __reduce__(self): - return (self.__class__, (list(self),), - getattr(self, '__dict__', None)) - - __hash__ = None - - def add(self, item): - if self._pending_removals: - self._commit_removals() - self.data.add(ref(item, self._remove)) - - def clear(self): - if self._pending_removals: - self._commit_removals() - self.data.clear() - - def copy(self): - return self.__class__(self) - - def pop(self): - if self._pending_removals: - self._commit_removals() - while True: - try: - itemref = self.data.pop() - except KeyError: - raise KeyError('pop from empty WeakSet') - item = itemref() - if item is not None: - return item - - def remove(self, item): - if self._pending_removals: - self._commit_removals() - self.data.remove(ref(item)) - - def discard(self, item): - if self._pending_removals: - self._commit_removals() - self.data.discard(ref(item)) - - def update(self, other): - if self._pending_removals: - self._commit_removals() - if isinstance(other, self.__class__): - self.data.update(other.data) - else: - for element in other: - self.add(element) - - def __ior__(self, other): - self.update(other) - return self - - # Helper functions for simple delegating methods. - def _apply(self, other, method): - if not isinstance(other, self.__class__): - other = self.__class__(other) - newdata = method(other.data) - newset = self.__class__() - newset.data = newdata - return newset - - def difference(self, other): - return self._apply(other, self.data.difference) - __sub__ = difference - - def difference_update(self, other): - if self._pending_removals: - self._commit_removals() - if self is other: - self.data.clear() - else: - self.data.difference_update(ref(item) for item in other) - def __isub__(self, other): - if self._pending_removals: - self._commit_removals() - if self is other: - self.data.clear() - else: - self.data.difference_update(ref(item) for item in other) - return self - - def intersection(self, other): - return self._apply(other, self.data.intersection) - __and__ = intersection - - def intersection_update(self, other): - if self._pending_removals: - self._commit_removals() - self.data.intersection_update(ref(item) for item in other) - def __iand__(self, other): - if self._pending_removals: - self._commit_removals() - self.data.intersection_update(ref(item) for item in other) - return self - - def issubset(self, other): - return self.data.issubset(ref(item) for item in other) - __lt__ = issubset - - def __le__(self, other): - return self.data <= set(ref(item) for item in other) - - def issuperset(self, other): - return self.data.issuperset(ref(item) for item in other) - __gt__ = issuperset - - def __ge__(self, other): - return self.data >= set(ref(item) for item in other) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return NotImplemented - return self.data == set(ref(item) for item in other) - - def symmetric_difference(self, other): - return self._apply(other, self.data.symmetric_difference) - __xor__ = symmetric_difference - - def symmetric_difference_update(self, other): - if self._pending_removals: - self._commit_removals() - if self is other: - self.data.clear() - else: - self.data.symmetric_difference_update(ref(item) for item in other) - def __ixor__(self, other): - if self._pending_removals: - self._commit_removals() - if self is other: - self.data.clear() - else: - self.data.symmetric_difference_update(ref(item) for item in other) - return self - - def union(self, other): - return self._apply(other, self.data.union) - __or__ = union - - def isdisjoint(self, other): - return len(self.intersection(other)) == 0 - - -# Instance of old-style class -class _C: pass -_InstanceType = type(_C()) - - -def abstractmethod(funcobj): - """A decorator indicating abstract methods. - - Requires that the metaclass is ABCMeta or derived from it. A - class that has a metaclass derived from ABCMeta cannot be - instantiated unless all of its abstract methods are overridden. - The abstract methods can be called using any of the normal - 'super' call mechanisms. - - Usage: - - class C: - __metaclass__ = ABCMeta - @abstractmethod - def my_abstract_method(self, ...): - ... - """ - funcobj.__isabstractmethod__ = True - return funcobj - - -class abstractproperty(property): - """A decorator indicating abstract properties. - - Requires that the metaclass is ABCMeta or derived from it. A - class that has a metaclass derived from ABCMeta cannot be - instantiated unless all of its abstract properties are overridden. - The abstract properties can be called using any of the normal - 'super' call mechanisms. - - Usage: - - class C: - __metaclass__ = ABCMeta - @abstractproperty - def my_abstract_property(self): - ... - - This defines a read-only property; you can also define a read-write - abstract property using the 'long' form of property declaration: - - class C: - __metaclass__ = ABCMeta - def getx(self): ... - def setx(self, value): ... - x = abstractproperty(getx, setx) - """ - __isabstractmethod__ = True - - -class ABCMeta(type): - - """Metaclass for defining Abstract Base Classes (ABCs). - - Use this metaclass to create an ABC. An ABC can be subclassed - directly, and then acts as a mix-in class. You can also register - unrelated concrete classes (even built-in classes) and unrelated - ABCs as 'virtual subclasses' -- these and their descendants will - be considered subclasses of the registering ABC by the built-in - issubclass() function, but the registering ABC won't show up in - their MRO (Method Resolution Order) nor will method - implementations defined by the registering ABC be callable (not - even via super()). - - """ - - # A global counter that is incremented each time a class is - # registered as a virtual subclass of anything. It forces the - # negative cache to be cleared before its next use. - _abc_invalidation_counter = 0 - - def __new__(mcls, name, bases, namespace): - cls = super(ABCMeta, mcls).__new__(mcls, name, bases, namespace) - # Compute set of abstract method names - abstracts = set(name - for name, value in namespace.items() - if getattr(value, "__isabstractmethod__", False)) - for base in bases: - for name in getattr(base, "__abstractmethods__", set()): - value = getattr(cls, name, None) - if getattr(value, "__isabstractmethod__", False): - abstracts.add(name) - cls.__abstractmethods__ = frozenset(abstracts) - # Set up inheritance registry - cls._abc_registry = WeakSet() - cls._abc_cache = WeakSet() - cls._abc_negative_cache = WeakSet() - cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter - return cls - - def register(cls, subclass): - """Register a virtual subclass of an ABC.""" - if not isinstance(subclass, (type, types.ClassType)): - raise TypeError("Can only register classes") - if issubclass(subclass, cls): - return # Already a subclass - # Subtle: test for cycles *after* testing for "already a subclass"; - # this means we allow X.register(X) and interpret it as a no-op. - if issubclass(cls, subclass): - # This would create a cycle, which is bad for the algorithm below - raise RuntimeError("Refusing to create an inheritance cycle") - cls._abc_registry.add(subclass) - ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache - - def _dump_registry(cls, file=None): - """Debug helper to print the ABC registry.""" - print >> file, "Class: %s.%s" % (cls.__module__, cls.__name__) - print >> file, "Inv.counter: %s" % ABCMeta._abc_invalidation_counter - for name in sorted(cls.__dict__.keys()): - if name.startswith("_abc_"): - value = getattr(cls, name) - print >> file, "%s: %r" % (name, value) - - def __instancecheck__(cls, instance): - """Override for isinstance(instance, cls).""" - # Inline the cache checking when it's simple. - subclass = getattr(instance, '__class__', None) - if subclass is not None and subclass in cls._abc_cache: - return True - subtype = type(instance) - # Old-style instances - if subtype is _InstanceType: - subtype = subclass - if subtype is subclass or subclass is None: - if (cls._abc_negative_cache_version == - ABCMeta._abc_invalidation_counter and - subtype in cls._abc_negative_cache): - return False - # Fall back to the subclass check. - return cls.__subclasscheck__(subtype) - return (cls.__subclasscheck__(subclass) or - cls.__subclasscheck__(subtype)) - - def __subclasscheck__(cls, subclass): - """Override for issubclass(subclass, cls).""" - # Check cache - if subclass in cls._abc_cache: - return True - # Check negative cache; may have to invalidate - if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter: - # Invalidate the negative cache - cls._abc_negative_cache = WeakSet() - cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter - elif subclass in cls._abc_negative_cache: - return False - # Check the subclass hook - ok = cls.__subclasshook__(subclass) - if ok is not NotImplemented: - assert isinstance(ok, bool) - if ok: - cls._abc_cache.add(subclass) - else: - cls._abc_negative_cache.add(subclass) - return ok - # Check if it's a direct subclass - if cls in getattr(subclass, '__mro__', ()): - cls._abc_cache.add(subclass) - return True - # Check if it's a subclass of a registered class (recursive) - for rcls in cls._abc_registry: - if issubclass(subclass, rcls): - cls._abc_cache.add(subclass) - return True - # Check if it's a subclass of a subclass (recursive) - for scls in cls.__subclasses__(): - if issubclass(subclass, scls): - cls._abc_cache.add(subclass) - return True - # No dice; update negative cache - cls._abc_negative_cache.add(subclass) - return False - - - - - - - - -import sys - -### ONE-TRICK PONIES ### - -def _hasattr(C, attr): - try: - return any(attr in B.__dict__ for B in C.__mro__) - except AttributeError: - # Old-style class - return hasattr(C, attr) - - -class Hashable: - __metaclass__ = ABCMeta - - @abstractmethod - def __hash__(self): - return 0 - - @classmethod - def __subclasshook__(cls, C): - if cls is Hashable: - try: - for B in C.__mro__: - if "__hash__" in B.__dict__: - if B.__dict__["__hash__"]: - return True - break - except AttributeError: - # Old-style class - if getattr(C, "__hash__", None): - return True - return NotImplemented - - -class Iterable: - __metaclass__ = ABCMeta - - @abstractmethod - def __iter__(self): - while False: - yield None - - @classmethod - def __subclasshook__(cls, C): - if cls is Iterable: - if _hasattr(C, "__iter__"): - return True - return NotImplemented - -Iterable.register(str) - - -class Iterator(Iterable): - - @abstractmethod - def next(self): - raise StopIteration - - def __iter__(self): - return self - - @classmethod - def __subclasshook__(cls, C): - if cls is Iterator: - if _hasattr(C, "next") and _hasattr(C, "__iter__"): - return True - return NotImplemented - - -class Sized: - __metaclass__ = ABCMeta - - @abstractmethod - def __len__(self): - return 0 - - @classmethod - def __subclasshook__(cls, C): - if cls is Sized: - if _hasattr(C, "__len__"): - return True - return NotImplemented - - -class Container: - __metaclass__ = ABCMeta - - @abstractmethod - def __contains__(self, x): - return False - - @classmethod - def __subclasshook__(cls, C): - if cls is Container: - if _hasattr(C, "__contains__"): - return True - return NotImplemented - - -class Callable: - __metaclass__ = ABCMeta - - @abstractmethod - def __call__(self, *args, **kwds): - return False - - @classmethod - def __subclasshook__(cls, C): - if cls is Callable: - if _hasattr(C, "__call__"): - return True - return NotImplemented - - -### SETS ### - - -class Set(Sized, Iterable, Container): - """A set is a finite, iterable container. - - This class provides concrete generic implementations of all - methods except for __contains__, __iter__ and __len__. - - To override the comparisons (presumably for speed, as the - semantics are fixed), all you have to do is redefine __le__ and - then the other operations will automatically follow suit. - """ - - def __le__(self, other): - if not isinstance(other, Set): - return NotImplemented - if len(self) > len(other): - return False - for elem in self: - if elem not in other: - return False - return True - - def __lt__(self, other): - if not isinstance(other, Set): - return NotImplemented - return len(self) < len(other) and self.__le__(other) - - def __gt__(self, other): - if not isinstance(other, Set): - return NotImplemented - return other < self - - def __ge__(self, other): - if not isinstance(other, Set): - return NotImplemented - return other <= self - - def __eq__(self, other): - if not isinstance(other, Set): - return NotImplemented - return len(self) == len(other) and self.__le__(other) - - def __ne__(self, other): - return not (self == other) - - @classmethod - def _from_iterable(cls, it): - '''Construct an instance of the class from any iterable input. - - Must override this method if the class constructor signature - does not accept an iterable for an input. - ''' - return cls(it) - - def __and__(self, other): - if not isinstance(other, Iterable): - return NotImplemented - return self._from_iterable(value for value in other if value in self) - - def isdisjoint(self, other): - for value in other: - if value in self: - return False - return True - - def __or__(self, other): - if not isinstance(other, Iterable): - return NotImplemented - chain = (e for s in (self, other) for e in s) - return self._from_iterable(chain) - - def __sub__(self, other): - if not isinstance(other, Set): - if not isinstance(other, Iterable): - return NotImplemented - other = self._from_iterable(other) - return self._from_iterable(value for value in self - if value not in other) - - def __xor__(self, other): - if not isinstance(other, Set): - if not isinstance(other, Iterable): - return NotImplemented - other = self._from_iterable(other) - return (self - other) | (other - self) - - # Sets are not hashable by default, but subclasses can change this - __hash__ = None - - def _hash(self): - """Compute the hash value of a set. - - Note that we don't define __hash__: not all sets are hashable. - But if you define a hashable set type, its __hash__ should - call this function. - - This must be compatible __eq__. - - All sets ought to compare equal if they contain the same - elements, regardless of how they are implemented, and - regardless of the order of the elements; so there's not much - freedom for __eq__ or __hash__. We match the algorithm used - by the built-in frozenset type. - """ - MAX = sys.maxint - MASK = 2 * MAX + 1 - n = len(self) - h = 1927868237 * (n + 1) - h &= MASK - for x in self: - hx = hash(x) - h ^= (hx ^ (hx << 16) ^ 89869747) * 3644798167 - h &= MASK - h = h * 69069 + 907133923 - h &= MASK - if h > MAX: - h -= MASK + 1 - if h == -1: - h = 590923713 - return h - -Set.register(frozenset) - - -class MutableSet(Set): - - @abstractmethod - def add(self, value): - """Add an element.""" - raise NotImplementedError - - @abstractmethod - def discard(self, value): - """Remove an element. Do not raise an exception if absent.""" - raise NotImplementedError - - def remove(self, value): - """Remove an element. If not a member, raise a KeyError.""" - if value not in self: - raise KeyError(value) - self.discard(value) - - def pop(self): - """Return the popped value. Raise KeyError if empty.""" - it = iter(self) - try: - value = next(it) - except StopIteration: - raise KeyError - self.discard(value) - return value - - def clear(self): - """This is slow (creates N new iterators!) but effective.""" - try: - while True: - self.pop() - except KeyError: - pass - - def __ior__(self, it): - for value in it: - self.add(value) - return self - - def __iand__(self, it): - for value in (self - it): - self.discard(value) - return self - - def __ixor__(self, it): - if it is self: - self.clear() - else: - if not isinstance(it, Set): - it = self._from_iterable(it) - for value in it: - if value in self: - self.discard(value) - else: - self.add(value) - return self - - def __isub__(self, it): - if it is self: - self.clear() - else: - for value in it: - self.discard(value) - return self - -MutableSet.register(set) - - -### MAPPINGS ### - - -class Mapping(Sized, Iterable, Container): - - @abstractmethod - def __getitem__(self, key): - raise KeyError - - def get(self, key, default=None): - try: - return self[key] - except KeyError: - return default - - def __contains__(self, key): - try: - self[key] - except KeyError: - return False - else: - return True - - def iterkeys(self): - return iter(self) - - def itervalues(self): - for key in self: - yield self[key] - - def iteritems(self): - for key in self: - yield (key, self[key]) - - def keys(self): - return list(self) - - def items(self): - return [(key, self[key]) for key in self] - - def values(self): - return [self[key] for key in self] - - # Mappings are not hashable by default, but subclasses can change this - __hash__ = None - - def __eq__(self, other): - if not isinstance(other, Mapping): - return NotImplemented - return dict(self.items()) == dict(other.items()) - - def __ne__(self, other): - return not (self == other) - -class MappingView(Sized): - - def __init__(self, mapping): - self._mapping = mapping - - def __len__(self): - return len(self._mapping) - - def __repr__(self): - return '{0.__class__.__name__}({0._mapping!r})'.format(self) - - -class KeysView(MappingView, Set): - - @classmethod - def _from_iterable(self, it): - return set(it) - - def __contains__(self, key): - return key in self._mapping - - def __iter__(self): - for key in self._mapping: - yield key - - -class ItemsView(MappingView, Set): - - @classmethod - def _from_iterable(self, it): - return set(it) - - def __contains__(self, item): - key, value = item - try: - v = self._mapping[key] - except KeyError: - return False - else: - return v == value - - def __iter__(self): - for key in self._mapping: - yield (key, self._mapping[key]) - - -class ValuesView(MappingView): - - def __contains__(self, value): - for key in self._mapping: - if value == self._mapping[key]: - return True - return False - - def __iter__(self): - for key in self._mapping: - yield self._mapping[key] - - -class MutableMapping(Mapping): - - @abstractmethod - def __setitem__(self, key, value): - raise KeyError - - @abstractmethod - def __delitem__(self, key): - raise KeyError - - __marker = object() - - def pop(self, key, default=__marker): - try: - value = self[key] - except KeyError: - if default is self.__marker: - raise - return default - else: - del self[key] - return value - - def popitem(self): - try: - key = next(iter(self)) - except StopIteration: - raise KeyError - value = self[key] - del self[key] - return key, value - - def clear(self): - try: - while True: - self.popitem() - except KeyError: - pass - - def update(*args, **kwds): - if len(args) > 2: - raise TypeError("update() takes at most 2 positional " - "arguments ({} given)".format(len(args))) - elif not args: - raise TypeError("update() takes at least 1 argument (0 given)") - self = args[0] - other = args[1] if len(args) >= 2 else () - - if isinstance(other, Mapping): - for key in other: - self[key] = other[key] - elif hasattr(other, "keys"): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - def setdefault(self, key, default=None): - try: - return self[key] - except KeyError: - self[key] = default - return default - -MutableMapping.register(dict) - - -### SEQUENCES ### - - -class Sequence(Sized, Iterable, Container): - """All the operations on a read-only sequence. - - Concrete subclasses must override __new__ or __init__, - __getitem__, and __len__. - """ - - @abstractmethod - def __getitem__(self, index): - raise IndexError - - def __iter__(self): - i = 0 - try: - while True: - v = self[i] - yield v - i += 1 - except IndexError: - return - - def __contains__(self, value): - for v in self: - if v == value: - return True - return False - - def __reversed__(self): - for i in reversed(range(len(self))): - yield self[i] - - def index(self, value): - for i, v in enumerate(self): - if v == value: - return i - raise ValueError - - def count(self, value): - return sum(1 for v in self if v == value) - -Sequence.register(tuple) -Sequence.register(basestring) -Sequence.register(buffer) -Sequence.register(xrange) - - -class MutableSequence(Sequence): - - @abstractmethod - def __setitem__(self, index, value): - raise IndexError - - @abstractmethod - def __delitem__(self, index): - raise IndexError - - @abstractmethod - def insert(self, index, value): - raise IndexError - - def append(self, value): - self.insert(len(self), value) - - def reverse(self): - n = len(self) - for i in range(n//2): - self[i], self[n-i-1] = self[n-i-1], self[i] - - def extend(self, values): - for v in values: - self.append(v) - - def pop(self, index=-1): - v = self[index] - del self[index] - return v - - def remove(self, value): - del self[self.index(value)] - - def __iadd__(self, values): - self.extend(values) - return self - -MutableSequence.register(list) - diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py old mode 100755 new mode 100644 diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py old mode 100755 new mode 100644 index ccc39734..00b2cd58 --- a/requests/packages/urllib3/_collections.py +++ b/requests/packages/urllib3/_collections.py @@ -4,14 +4,9 @@ # This module is part of urllib3 and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php - from collections import deque -from threading import RLock -try: - from collections import MutableMapping -except ImportError: - from .__collections import MutableMapping +from threading import RLock __all__ = ['RecentlyUsedContainer'] @@ -24,7 +19,7 @@ class AccessEntry(object): self.is_valid = is_valid -class RecentlyUsedContainer(MutableMapping): +class RecentlyUsedContainer(dict): """ Provides a dict-like that maintains up to ``maxsize`` keys while throwing away the least-recently-used keys beyond ``maxsize``. @@ -81,7 +76,7 @@ class RecentlyUsedContainer(MutableMapping): if not p.is_valid: continue # Invalidated entry, skip - self._container.pop(p.key, None) + dict.pop(self, p.key, None) self.access_lookup.pop(p.key, None) num -= 1 @@ -100,7 +95,7 @@ class RecentlyUsedContainer(MutableMapping): return r def __getitem__(self, key): - item = self._container.get(key) + item = dict.get(self, key) if not item: raise KeyError(key) @@ -118,22 +113,19 @@ class RecentlyUsedContainer(MutableMapping): def __setitem__(self, key, item): # Add item to our container and access log - self._container[key] = item + dict.__setitem__(self, key, item) self._push_entry(key) # Discard invalid and excess entries - self._prune_entries(len(self._container) - self._maxsize) + self._prune_entries(len(self) - self._maxsize) def __delitem__(self, key): self._invalidate_entry(key) - del self._container[key] - del self.access_lookup[key] + self.access_lookup.pop(key, None) + dict.__delitem__(self, key) - def __len__(self): - return self._container.__len__() - - def __iter__(self): - return self._container.__iter__() - - def __contains__(self, key): - return self._container.__contains__(key) + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py old mode 100755 new mode 100644 index 8b10dc70..4c2c6b54 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -338,7 +338,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): except (SocketTimeout, Empty), e: # Timed out either by socket or queue - raise TimeoutError("Request timed out after %s seconds" % + raise TimeoutError("Request timed out after %f seconds" % self.timeout) except (BaseSSLError), e: diff --git a/requests/packages/urllib3/contrib/__init__.py b/requests/packages/urllib3/contrib/__init__.py old mode 100755 new mode 100644 diff --git a/requests/packages/urllib3/contrib/ntlmpool.py b/requests/packages/urllib3/contrib/ntlmpool.py old mode 100755 new mode 100644 diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py old mode 100755 new mode 100644 diff --git a/requests/packages/urllib3/filepost.py b/requests/packages/urllib3/filepost.py old mode 100755 new mode 100644 diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py old mode 100755 new mode 100644 diff --git a/requests/packages/urllib3/request.py b/requests/packages/urllib3/request.py old mode 100755 new mode 100644 diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py old mode 100755 new mode 100644 From bcc48a9607daf87a90bc2a70d9bfaac79b56d80c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 13 Nov 2011 00:43:14 -0500 Subject: [PATCH 254/255] whatespace --- requests/models.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index 79a15fc2..34de8005 100644 --- a/requests/models.py +++ b/requests/models.py @@ -1,4 +1,3 @@ - # -*- coding: utf-8 -*- """ @@ -31,8 +30,6 @@ from .utils import ( decode_gzip, stream_decode_gzip, guess_filename) - - REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) From 9aa62d5622fd69bc77bad3ea24ed8379eb158de8 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 13 Nov 2011 00:48:54 -0500 Subject: [PATCH 255/255] 13th --- HISTORY.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0648118d..05ba2f49 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,7 +2,7 @@ History ------- -0.8.0 (2011-11-09) +0.8.0 (2011-11-13) ++++++++++++++++++ * Keep-alive support! @@ -14,6 +14,7 @@ History * prefetch parameter for request methods * OPTION method * Async pool size throttling +* File uploads send real names 0.7.6 (2011-11-07) ++++++++++++++++++