From c2a1f28a2ebd9448be7b4c8f9207d16c894717c3 Mon Sep 17 00:00:00 2001 From: schlamar Date: Mon, 10 Mar 2014 07:54:22 +0100 Subject: [PATCH 1/4] Catch possible exceptions while consuming content of redirect responses. --- requests/sessions.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index 6b21b5df..fe8d7e2a 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -19,7 +19,8 @@ from .cookies import ( from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT from .hooks import default_hooks, dispatch_hook from .utils import to_key_val_list, default_headers, to_native_string -from .exceptions import TooManyRedirects, InvalidSchema +from .exceptions import ( + TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) from .structures import CaseInsensitiveDict from .adapters import HTTPAdapter @@ -94,7 +95,10 @@ class SessionRedirectMixin(object): while resp.is_redirect: prepared_request = req.copy() - resp.content # Consume socket so it can be released + try: + resp.content # Consume socket so it can be released + except (ChunkedEncodingError, ContentDecodingError, RuntimeError): + resp.raw.read(decode_content=False) if i >= self.max_redirects: raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects) From 59c8d813818110aac29fd104c2fa012387c2004c Mon Sep 17 00:00:00 2001 From: schlamar Date: Mon, 10 Mar 2014 08:07:10 +0100 Subject: [PATCH 2/4] Read content in Session.send instead of Adapter.send. --- requests/adapters.py | 7 +------ requests/sessions.py | 3 +++ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 12d23b01..eb7a2d28 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -385,9 +385,4 @@ class HTTPAdapter(BaseAdapter): else: raise - r = self.build_response(request, resp) - - if not stream: - r.content - - return r + return self.build_response(request, resp) diff --git a/requests/sessions.py b/requests/sessions.py index fe8d7e2a..df85a25c 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -592,6 +592,9 @@ class Session(SessionRedirectMixin): r = history.pop() r.history = history + if not stream: + r.content + return r def get_adapter(self, url): From 5e860c08d8b3f14bcdf0382b0535da39a4d5a02e Mon Sep 17 00:00:00 2001 From: schlamar Date: Thu, 13 Mar 2014 18:27:12 +0100 Subject: [PATCH 3/4] Added test case to handle response of streamed redirects. Credits go to @zackw. --- test_requests.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/test_requests.py b/test_requests.py index a1df1f19..e3eacd6f 100755 --- a/test_requests.py +++ b/test_requests.py @@ -917,6 +917,25 @@ class RequestsTestCase(unittest.TestCase): assert h1 == h2 + def test_manual_redirect_with_partial_body_read(self): + s = requests.Session() + r1 = s.get(httpbin('redirect/2'), allow_redirects=False, stream=True) + assert r1.is_redirect + rg = s.resolve_redirects(r1, r1.request, stream=True) + + # read only the first eight bytes of the response body, + # then follow the redirect + r1.iter_content(8) + r2 = next(rg) + assert r2.is_redirect + + # read all of the response via iter_content, + # then follow the redirect + for _ in r2.iter_content(): + pass + r3 = next(rg) + assert not r3.is_redirect + class TestContentEncodingDetection(unittest.TestCase): @@ -1321,7 +1340,7 @@ def test_data_argument_accepts_tuples(list_of_tuples): hooks=default_hooks() ) assert p.body == urlencode(data) - + if __name__ == '__main__': unittest.main() From 16459910a963c108529585fe8feb78f862d6d9ba Mon Sep 17 00:00:00 2001 From: schlamar Date: Sat, 15 Mar 2014 21:29:14 +0100 Subject: [PATCH 4/4] Added test for redirect with wrong gzipped header. --- test_requests.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test_requests.py b/test_requests.py index e3eacd6f..a92b22c6 100755 --- a/test_requests.py +++ b/test_requests.py @@ -936,6 +936,26 @@ class RequestsTestCase(unittest.TestCase): r3 = next(rg) assert not r3.is_redirect + def _patch_adapter_gzipped_redirect(self, session, url): + adapter = session.get_adapter(url=url) + org_build_response = adapter.build_response + self._patched_response = False + + def build_response(*args, **kwargs): + resp = org_build_response(*args, **kwargs) + if not self._patched_response: + resp.raw.headers['content-encoding'] = 'gzip' + self._patched_response = True + return resp + + adapter.build_response = build_response + + def test_redirect_with_wrong_gzipped_header(self): + s = requests.Session() + url = httpbin('redirect/1') + self._patch_adapter_gzipped_redirect(s, url) + s.get(url) + class TestContentEncodingDetection(unittest.TestCase):