From 0b6dc091f50965250d9c421d8aadaa701186a1b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Mon, 8 Aug 2011 18:11:33 +0200 Subject: [PATCH 01/22] Added support for URLs with path not encoded. --- requests/models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index a67691dd..bb15dc9b 100644 --- a/requests/models.py +++ b/requests/models.py @@ -185,7 +185,7 @@ class Request(object): # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') if not urlparse(url).netloc: parent_url_components = urlparse(self.url) - url = '%s://%s/%s' % (parent_url_components.scheme, parent_url_components.netloc, url) + url = '%s://%s/%s' % (parent_url_components.scheme, parent_url_components.netloc, urllib.quote(urllib.unquote(url))) # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 if r.status_code is 303: @@ -231,9 +231,10 @@ class Request(object): def _build_url(self): """Build the actual URL to use""" - # Support for unicode domain names. + # Support for unicode domain names and paths. parsed_url = list(urlparse(self.url)) parsed_url[1] = parsed_url[1].encode('idna') + parsed_url[2] = urllib.quote(urllib.unquote(parsed_url[2])) self.url = urlunparse(parsed_url) if self._enc_params: From aa0f78b740b6098f29594242b6b923113d0659b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Tue, 9 Aug 2011 15:14:08 +0200 Subject: [PATCH 02/22] Better support of utf-8 paths. --- requests/models.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/requests/models.py b/requests/models.py index 619d099f..51100290 100644 --- a/requests/models.py +++ b/requests/models.py @@ -247,10 +247,12 @@ class Request(object): """Build the actual URL to use""" # Support for unicode domain names and paths. - parsed_url = list(urlparse(self.url)) - parsed_url[1] = parsed_url[1].encode('idna') - parsed_url[2] = urllib.quote(urllib.unquote(parsed_url[2])) - self.url = urlunparse(parsed_url) + scheme, netloc, path, params, query, fragment = urlparse(self.url) + netloc = netloc.encode('idna') + if isinstance(path, unicode): + path = path.encode('utf-8') + path = urllib.quote(urllib.unquote(path)) + self.url = str(urlunparse([ scheme, netloc, path, params, query, fragment ])) if self._enc_params: if urlparse(self.url).query: From 32a861b2eddcfd73049f245984a12b88b364dadd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Tue, 9 Aug 2011 15:34:44 +0200 Subject: [PATCH 03/22] Fixed memory leak (see http://bugs.python.org/issue1208304) --- requests/models.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 51100290..6d97e35e 100644 --- a/requests/models.py +++ b/requests/models.py @@ -169,7 +169,8 @@ class Request(object): try: response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None)) response.read = resp.read - response.close = resp.close + response._resp = resp + response._close = resp.close except AttributeError: pass @@ -397,6 +398,11 @@ class Response(object): raise self.error + def close(self): + if self._resp.fp is not None and hasattr(self._resp.fp, '_sock'): + self._resp.fp._sock.recv = None + self._close() + class AuthManager(object): """Requests Authentication Manager.""" From 7b79cea7384a2dd3df90ac748bac3c4a89ebce24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Tue, 9 Aug 2011 15:41:59 +0200 Subject: [PATCH 04/22] Use urljoin instead of manual concatenation. --- requests/models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index 51100290..57ee5dd8 100644 --- a/requests/models.py +++ b/requests/models.py @@ -12,7 +12,7 @@ import socket import zlib from urllib2 import HTTPError -from urlparse import urlparse, urlunparse +from urlparse import urlparse, urlunparse, urljoin from datetime import datetime from .config import settings @@ -198,8 +198,7 @@ class Request(object): # Facilitate non-RFC2616-compliant 'location' headers # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') if not urlparse(url).netloc: - parent_url_components = urlparse(self.url) - url = '%s://%s/%s' % (parent_url_components.scheme, parent_url_components.netloc, urllib.quote(urllib.unquote(url))) + url = urljoin(r.url, urllib.quote(urllib.unquote(url))) # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 if r.status_code is 303: From e7e395549f8b60777f00c244dabd2bd9f89271b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Tue, 9 Aug 2011 17:28:58 +0200 Subject: [PATCH 05/22] Handle too many redirects. --- requests/exceptions.py | 3 +++ requests/models.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/requests/exceptions.py b/requests/exceptions.py index eff7512a..c08c6148 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -21,3 +21,6 @@ class URLRequired(RequestException): class InvalidMethod(RequestException): """An inappropriate method was attempted.""" + +class TooManyRedirects(RequestException): + """Too many redirects.""" diff --git a/requests/models.py b/requests/models.py index ca72802f..e08bce25 100644 --- a/requests/models.py +++ b/requests/models.py @@ -20,7 +20,7 @@ from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicA from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers -from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod +from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects REDIRECT_STATI = (301, 302, 303, 307) @@ -192,6 +192,9 @@ class Request(object): (self.allow_redirects)) ): + if not len(history) < 30: + raise TooManyRedirects() + history.append(r) url = r.headers['location'] From cf94c96a68904b0c541425bb2eb0ee816d6a4e4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Tue, 9 Aug 2011 17:30:19 +0200 Subject: [PATCH 06/22] Always close connection during redirections. --- requests/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requests/models.py b/requests/models.py index 6d97e35e..926bb94b 100644 --- a/requests/models.py +++ b/requests/models.py @@ -192,6 +192,8 @@ class Request(object): (self.allow_redirects)) ): + r.close() + history.append(r) url = r.headers['location'] From 91e75f721508d2198ccb3d4e2979047249ae01f4 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Wed, 17 Aug 2011 13:40:29 +0300 Subject: [PATCH 07/22] As proposed by @davidbgk this is only consistent with \o/ :) --- requests/status_codes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requests/status_codes.py b/requests/status_codes.py index e8023142..c8a47a07 100644 --- a/requests/status_codes.py +++ b/requests/status_codes.py @@ -22,7 +22,7 @@ _codes = { # Redirection. 300: ('multiple_choices',), - 301: ('moved_permanently', 'moved'), + 301: ('moved_permanently', 'moved', '\\o-'), 302: ('found',), 302: ('see_other', 'other'), 304: ('not_modified',), @@ -36,7 +36,7 @@ _codes = { 401: ('unauthorized',), 402: ('payment_required', 'payment'), 403: ('forbidden',), - 404: ('not_found',), + 404: ('not_found', '-o-'), 405: ('method_not_allowed', 'not_allowed'), 406: ('not_acceptable',), 407: ('proxy_authentication_required', 'proxy_auth', 'proxy_authentication'), @@ -62,7 +62,7 @@ _codes = { 499: ('client_closed_request',), # Server Error. - 500: ('internal_server_error', 'server_error'), + 500: ('internal_server_error', 'server_error', '/o\\'), 501: ('not_implemented',), 502: ('bad_gateway',), 503: ('service_unavailable', 'unavailable'), From 5e6ecdad9f69b1ff789a17733b8edc6fd7091bd8 Mon Sep 17 00:00:00 2001 From: Shrikant Sharat Kandula Date: Thu, 8 Sep 2011 02:38:50 +51800 Subject: [PATCH 08/22] Typo in documentation The kwarg is named `headers`, not `header`. Also, its a dict, not a set. --- docs/user/advanced.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index cdf1abb6..d145da21 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -34,7 +34,7 @@ Sessions can also be used to provide default data to the request methods:: with requests.session(auth=auth, headers=headers) as c: # both 'x-test' and 'x-test2' are sent - c.get('http://httpbin.org/headers', header={'x-test2', 'true'}) + c.get('http://httpbin.org/headers', headers={'x-test2': 'true'}) .. admonition:: Global Settings From 571d808d25d3391664a69920d4e0a5cb1418af90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Wed, 17 Aug 2011 15:31:24 +0200 Subject: [PATCH 09/22] Fixed bad merge. --- requests/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 0d7b67cb..7690f16a 100644 --- a/requests/models.py +++ b/requests/models.py @@ -230,7 +230,8 @@ class Request(object): # Facilitate non-RFC2616-compliant 'location' headers # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') - url = urljoin(r.url, urllib.quote(urllib.unquote(url))) + if not urlparse(url).netloc: + url = urljoin(r.url, urllib.quote(urllib.unquote(url))) # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 if r.status_code is 303: From 61d50d531cca57061335c776148c55d452ef5c3e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 09:41:24 -0400 Subject: [PATCH 10/22] Authors += Armin --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 2d34a221..dd1b3109 100644 --- a/AUTHORS +++ b/AUTHORS @@ -35,3 +35,4 @@ Patches and Suggestions - Jens Diemer - Alex <@alopatin> - Tom Hogans +- Armin Ronacher \ No newline at end of file From 007963afde4d856803e61357c64f29329a361ad9 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 09:45:31 -0400 Subject: [PATCH 11/22] authors --- AUTHORS.orig | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 AUTHORS.orig diff --git a/AUTHORS.orig b/AUTHORS.orig new file mode 100644 index 00000000..dd88224b --- /dev/null +++ b/AUTHORS.orig @@ -0,0 +1,40 @@ +Requests is written and maintained by Kenneth Reitz and +various contributors: + +Development Lead +```````````````` + +- Kenneth Reitz + + +Patches and Suggestions +``````````````````````` + +- Various Pocoo Members +- Chris Adams +- Flavio Percoco Premoli +- Dj Gilcrease +- Justin Murphy +- Rob Madole +- Aram Dulyan +- Johannes Gorset +- 村山めがね (Megane Murayama) +- James Rowe +- Daniel Schauenberg +- Zbigniew Siciarz +- Daniele Tricoli 'Eriol' +- Richard Boulton +- Miguel Olivares +- Alberto Paro +- Jérémy Bethmont +- 潘旭 (Xu Pan) +- Tamás Gulácsi +- Rubén Abad +- Peter Manser +- Jeremy Selie +<<<<<<< HEAD +- Jens Diemer +- Alex <@alopatin> +======= +- Tom Hogans +>>>>>>> 0ed641a26ec2200de00e4bbf3d170c767375351e From c22b71fc5b69736f40f8e491c5a6c4db461321f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Je=CC=81re=CC=81my=20Bethmont?= Date: Wed, 17 Aug 2011 16:04:53 +0200 Subject: [PATCH 12/22] Handle redirection without scheme. --- requests/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/requests/models.py b/requests/models.py index cc9cc6f0..931bafd8 100644 --- a/requests/models.py +++ b/requests/models.py @@ -225,6 +225,11 @@ class Request(object): url = r.headers['location'] + # Handle redirection without scheme (see: RFC 1808 Section 4) + if url.startswith('//'): + parsed_rurl = urlparse(r.url) + url = '%s:%s' % (parsed_rurl.scheme, url) + # Facilitate non-RFC2616-compliant 'location' headers # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') if not urlparse(url).netloc: From 96e7a60b8d8a930d3257b87e3159b5abbbf385d5 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 20:13:44 -0400 Subject: [PATCH 13/22] Allow any method, if so inclined #124 --- AUTHORS | 3 ++- requests/models.py | 10 ---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/AUTHORS b/AUTHORS index dd1b3109..5bd399a2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -35,4 +35,5 @@ Patches and Suggestions - Jens Diemer - Alex <@alopatin> - Tom Hogans -- Armin Ronacher \ No newline at end of file +- Armin Ronacher +- Shrikant Sharat Kandula \ No newline at end of file diff --git a/requests/models.py b/requests/models.py index d6b058fb..0b22f711 100644 --- a/requests/models.py +++ b/requests/models.py @@ -32,8 +32,6 @@ class Request(object): Requests. Recommended interface is with the Requests functions. """ - _METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'PATCH') - def __init__(self, url=None, headers=dict(), files=None, method=None, data=dict(), params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False, @@ -116,14 +114,6 @@ class Request(object): return '' % (self.method) - def __setattr__(self, name, value): - if (name == 'method') and (value): - if not value in self._METHODS: - raise InvalidMethod() - - object.__setattr__(self, name, value) - - def _checks(self): """Deterministic checks for consistency.""" From efbbb84315505d78aff709106c53073e5111a9fe Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 17 Aug 2011 20:17:22 -0400 Subject: [PATCH 14/22] Added Mikko Ohtamaa to AUTHORS #124 --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 5bd399a2..ff0c44b6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -36,4 +36,5 @@ Patches and Suggestions - Alex <@alopatin> - Tom Hogans - Armin Ronacher -- Shrikant Sharat Kandula \ No newline at end of file +- Shrikant Sharat Kandula +- Mikko Ohtamaa \ No newline at end of file From 2f55393593fb0d8818fa014bc004dfed73564110 Mon Sep 17 00:00:00 2001 From: Den Shabalin Date: Fri, 19 Aug 2011 19:18:26 +0300 Subject: [PATCH 15/22] Fixes an issue #128: ``Response not working with lxml''. This error happend due to lxml's attempt to do .geturl() call on the response object. __getattr__ didn't raise AttributeError so ``response.geturl'' returned None and ``response.geturl()'' resulted into ``TypeError: 'NoneType' object is not callable'' seen in the issue. --- requests/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 0b22f711..5e0a4d3b 100644 --- a/requests/models.py +++ b/requests/models.py @@ -416,7 +416,8 @@ class Response(object): except zlib.error: pass return self._content - + else: + raise AttributeError def raise_for_status(self): """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured.""" From ce4927e5d36e02bfb9de064f87e6027fff8c1a1f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Aug 2011 19:39:53 -0400 Subject: [PATCH 16/22] added Den Shabalin to AUTHORS --- AUTHORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index ff0c44b6..61874982 100644 --- a/AUTHORS +++ b/AUTHORS @@ -37,4 +37,5 @@ Patches and Suggestions - Tom Hogans - Armin Ronacher - Shrikant Sharat Kandula -- Mikko Ohtamaa \ No newline at end of file +- Mikko Ohtamaa +- Den Shabalin \ No newline at end of file From d9a349c1a1451f7c6d1dbea0b637b35107eb8de5 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Aug 2011 20:12:16 -0400 Subject: [PATCH 17/22] 303 status code --- requests/status_codes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/status_codes.py b/requests/status_codes.py index c8a47a07..a809de6a 100644 --- a/requests/status_codes.py +++ b/requests/status_codes.py @@ -24,7 +24,7 @@ _codes = { 300: ('multiple_choices',), 301: ('moved_permanently', 'moved', '\\o-'), 302: ('found',), - 302: ('see_other', 'other'), + 303: ('see_other', 'other'), 304: ('not_modified',), 305: ('use_proxy',), 306: ('switch_proxy',), From c75eaec8527b32a5b1bdf95c1c70f34a33f508d6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Aug 2011 20:12:33 -0400 Subject: [PATCH 18/22] integrate codes into models.py --- requests/models.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/requests/models.py b/requests/models.py index 7e8d51b6..2d7fc8fe 100644 --- a/requests/models.py +++ b/requests/models.py @@ -22,9 +22,11 @@ from .packages.poster.encode import multipart_encode from .packages.poster.streaminghttp import register_openers, get_handlers from .utils import dict_from_cookiejar from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects +from .status_codes import codes -REDIRECT_STATI = (301, 302, 303, 307) +REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) + class Request(object): @@ -202,13 +204,13 @@ class Request(object): while ( ('location' in r.headers) and ((self.method in ('GET', 'HEAD')) or - (r.status_code is 303) or + (r.status_code is codes.see_other) or (self.allow_redirects)) ): r.close() - if not len(history) < 30: + if not len(history) < settings.max_redirects: raise TooManyRedirects() history.append(r) @@ -226,7 +228,7 @@ class Request(object): url = urljoin(r.url, urllib.quote(urllib.unquote(url))) # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 - if r.status_code is 303: + if r.status_code is codes.see_other: method = 'GET' else: method = self.method From abc5e279319425e226121165d15052542acfaa9a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Aug 2011 20:12:41 -0400 Subject: [PATCH 19/22] config max_redirects --- requests/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/requests/config.py b/requests/config.py index a42f15d2..39be2ed4 100644 --- a/requests/config.py +++ b/requests/config.py @@ -61,6 +61,7 @@ settings.accept_gzip = True settings.proxies = None settings.verbose = None settings.timeout = None +settings.max_redirects = 30 #: Use socket.setdefaulttimeout() as fallback? settings.timeout_fallback = True From 53a53668d90f4defdc5ece7ed79f7cbac8bed5f3 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Aug 2011 20:59:16 -0400 Subject: [PATCH 20/22] no more roadmap --- README.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.rst b/README.rst index 529d1445..3f6455ed 100644 --- a/README.rst +++ b/README.rst @@ -100,11 +100,5 @@ Contribute If you'd like to contribute, simply fork `the repository`_, commit your changes to the **develop** branch (or branch off of it), and send a pull request. Make sure you add yourself to AUTHORS_. - -Roadmap -------- - -- Sphinx Documentation - .. _`the repository`: http://github.com/kennethreitz/requests .. _AUTHORS: http://github.com/kennethreitz/requests/blob/master/AUTHORS From c58ddff3a4b782012894e148bf471e0ebd146d64 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 18:24:58 -0400 Subject: [PATCH 21/22] history update --- HISTORY.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index b2ae8f9f..b09392b5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,17 @@ History ------- +0.6.1 (2011-08-20) +++++++++++++++++++ + +* Enhanced status codes experience ``\o/`` +* Set a maximum number of redirects (``settings.max_redirects``) +* Full Unicode URL support +* Support for protocol-less redirects. +* Allow for arbitrary request types. +* Bugfixes + + 0.6.0 (2011-08-17) ++++++++++++++++++ From 8b640979ee06a03a57e72ba3ed73fcb73efc4c74 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 20 Aug 2011 18:25:32 -0400 Subject: [PATCH 22/22] v0.6.1 --- requests/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/core.py b/requests/core.py index 68e53923..8ba34a2f 100644 --- a/requests/core.py +++ b/requests/core.py @@ -12,8 +12,8 @@ This module implements the main Requests system. """ __title__ = 'requests' -__version__ = '0.6.0' -__build__ = 0x000600 +__version__ = '0.6.1' +__build__ = 0x000601 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz'