From 49bed18a8357cf60761e55bb88802233de78bbe3 Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Sun, 29 Jan 2012 16:51:28 -0500 Subject: [PATCH 1/6] Minor PEP-8 cleanup --- requests/models.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/requests/models.py b/requests/models.py index c2cbe979..3f8547ef 100644 --- a/requests/models.py +++ b/requests/models.py @@ -207,11 +207,8 @@ class Request(object): self.cookies.update(r.cookies) if r.status_code in REDIRECT_STATI and not self.redirect: - - while ( - ('location' in r.headers) and - ((r.status_code is codes.see_other) or (self.allow_redirects)) - ): + while (('location' in r.headers) and + ((r.status_code is codes.see_other) or (self.allow_redirects))): if not len(history) < self.config.get('max_redirects'): raise TooManyRedirects() From 3020d0e56a552cd0a1f603aba69393511ac22d81 Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Sun, 29 Jan 2012 16:52:08 -0500 Subject: [PATCH 2/6] Consistent self.cookies usage This avoids two variables pointing to the same thing and avoids a duplicate dict update() call --- requests/models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index 3f8547ef..b4118b93 100644 --- a/requests/models.py +++ b/requests/models.py @@ -203,7 +203,7 @@ class Request(object): history = [] r = build(resp) - cookies = self.cookies + self.cookies.update(r.cookies) if r.status_code in REDIRECT_STATI and not self.redirect: @@ -247,7 +247,7 @@ class Request(object): method=method, params=self.session.params, auth=self.auth, - cookies=cookies, + cookies=self.cookies, redirect=True, config=self.config, timeout=self.timeout, @@ -258,7 +258,6 @@ class Request(object): ) request.send() - cookies.update(request.response.cookies) r = request.response self.cookies.update(r.cookies) From 730c0e2e2bef77968a86962f9d5f2bebba4d19ec Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Sun, 29 Jan 2012 17:31:57 -0500 Subject: [PATCH 3/6] Initial stop-the-bleeding fix This is redundant but it avoids crashing otherwise working code. The code-smell from duplication suggests that this interface should be refactored slightly. --- requests/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requests/models.py b/requests/models.py index b4118b93..770e9b04 100644 --- a/requests/models.py +++ b/requests/models.py @@ -210,6 +210,8 @@ class Request(object): while (('location' in r.headers) and ((r.status_code is codes.see_other) or (self.allow_redirects))): + r.content # Consume socket so it can be released + if not len(history) < self.config.get('max_redirects'): raise TooManyRedirects() @@ -262,6 +264,7 @@ class Request(object): self.cookies.update(r.cookies) r.history = history + r.content # Consume socket so it can be released self.response = r self.response.request = self From 495646bb6a89802a60208c2ab09749edba1f8e51 Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Sun, 29 Jan 2012 18:42:46 -0500 Subject: [PATCH 4/6] async: force prefetch=True on send MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … otherwise we can run out of sockets before any of the responses are ever processed --- requests/async.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/requests/async.py b/requests/async.py index 94884478..24f8a617 100644 --- a/requests/async.py +++ b/requests/async.py @@ -46,15 +46,15 @@ def patched(f): return wrapped -def send(r, pool=None): - """Sends the request object using the specified pool. If a pool isn't +def send(r, pool=None, prefetch=False): + """Sends the request object using the specified pool. If a pool isn't specified this method blocks. Pools are useful because you can specify size and can hence limit concurrency.""" if pool != None: - return pool.spawn(r.send) + return pool.spawn(r.send, prefetch=prefetch) - return gevent.spawn(r.send) + return gevent.spawn(r.send, prefetch=prefetch) # Patched requests.api functions. @@ -79,10 +79,7 @@ def map(requests, prefetch=True, size=None): requests = list(requests) pool = Pool(size) if size else None - jobs = [send(r, pool) for r in requests] + jobs = [send(r, pool, prefetch=prefetch) for r in requests] gevent.joinall(jobs) - if prefetch: - [r.response.content for r in requests] - return [r.response for r in requests] \ No newline at end of file From b98cc8b4e9d8c4c12b8f52c9a9be151c306cb977 Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Sun, 29 Jan 2012 18:43:13 -0500 Subject: [PATCH 5/6] Revert "Initial stop-the-bleeding fix" This reverts commit 730c0e2e2bef77968a86962f9d5f2bebba4d19ec. --- requests/models.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/requests/models.py b/requests/models.py index 770e9b04..b4118b93 100644 --- a/requests/models.py +++ b/requests/models.py @@ -210,8 +210,6 @@ class Request(object): while (('location' in r.headers) and ((r.status_code is codes.see_other) or (self.allow_redirects))): - r.content # Consume socket so it can be released - if not len(history) < self.config.get('max_redirects'): raise TooManyRedirects() @@ -264,7 +262,6 @@ class Request(object): self.cookies.update(r.cookies) r.history = history - r.content # Consume socket so it can be released self.response = r self.response.request = self From 67f50a0fe53765987e5a6a1c02daabe6400322af Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Sun, 29 Jan 2012 18:45:14 -0500 Subject: [PATCH 6/6] Less redundant freeing of resources on redirects With 495646bb6a89802a60208c2ab09749edba1f8e51 the double flush is no longer needed. --- requests/models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requests/models.py b/requests/models.py index b4118b93..0647814f 100644 --- a/requests/models.py +++ b/requests/models.py @@ -210,6 +210,8 @@ class Request(object): while (('location' in r.headers) and ((r.status_code is codes.see_other) or (self.allow_redirects))): + r.content # Consume socket so it can be released + if not len(history) < self.config.get('max_redirects'): raise TooManyRedirects()