From c658b363e38c928a2f7be8e09eece8e11f863b62 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 17 Jan 2012 15:17:38 -0600 Subject: [PATCH 1/3] The async.send method wasn't really doing anything useful. It was just calling the r.send() method of the Request object 'r'. It took an optional kwarg pools, and assigned it to r._pools which isn't used by anything. This situation was eaither broken or not fully implemented yet. I modified async.map to call r.send rather than send(r) and then modified async.send to actually make use of the pool that it is passed. --- requests/async.py | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/requests/async.py b/requests/async.py index 6f8de380..36f6c5a9 100644 --- a/requests/async.py +++ b/requests/async.py @@ -46,15 +46,16 @@ def patched(f): return wrapped -def send(r, pools=None): - """Sends a given Request object.""" +def send(r, pool=None): + """Sends the request object using the specified pool. If a pool isn't + specified this method blocks. Pools are useful because you can specify size + and can hence limit concurrency.""" - if pools: - r._pools = pools + if pool != None: + print pool.full() + return pool.spawn(r.send) - r.send() - - return r.response + return gevent.spawn(r.send) # Patched requests.api functions. @@ -80,17 +81,13 @@ def map(requests, prefetch=True, size=None): if size: pool = Pool(size) - pool.map(send, requests) - pool.join() + jobs = [pool.spawn(r.send) for r in requests] else: - jobs = [gevent.spawn(send, r) for r in requests] - gevent.joinall(jobs) + jobs = [gevent.spawn(r.send) for r in requests] + + gevent.joinall(jobs) if prefetch: [r.response.content for r in requests] - return [r.response for r in requests] - - - - + return [r.response for r in requests] \ No newline at end of file From 97be9ee4cd6326bf7551f7a737877ef2e368ef20 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 17 Jan 2012 15:20:52 -0600 Subject: [PATCH 2/3] The async.send method wasn't really doing anything useful. It was just calling the r.send() method of the Request object 'r'. It took an optional kwarg pools, and assigned it to r._pools which isn't used by anything. This situation was eaither broken or not fully implemented yet. I modified async.map to call r.send rather than send(r) and then modified async.send to actually make use of the pool that it is passed. --- requests/async.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/async.py b/requests/async.py index 36f6c5a9..53aa8418 100644 --- a/requests/async.py +++ b/requests/async.py @@ -52,7 +52,7 @@ def send(r, pool=None): and can hence limit concurrency.""" if pool != None: - print pool.full() + return pool.spawn(r.send) return gevent.spawn(r.send) From e38046e6c65b8f03026d13d339a9bacf849b2d85 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 17 Jan 2012 15:21:09 -0600 Subject: [PATCH 3/3] The async.send method wasn't really doing anything useful. It was just calling the r.send() method of the Request object 'r'. It took an optional kwarg pools, and assigned it to r._pools which isn't used by anything. This situation was eaither broken or not fully implemented yet. I modified async.map to call r.send rather than send(r) and then modified async.send to actually make use of the pool that it is passed. --- requests/async.py | 1 - 1 file changed, 1 deletion(-) diff --git a/requests/async.py b/requests/async.py index 53aa8418..fafcb9ed 100644 --- a/requests/async.py +++ b/requests/async.py @@ -52,7 +52,6 @@ def send(r, pool=None): and can hence limit concurrency.""" if pool != None: - return pool.spawn(r.send) return gevent.spawn(r.send)