From 2eb682671db4cdcc1fceac0240cb57683f9497ba Mon Sep 17 00:00:00 2001 From: Randy Merrill Date: Fri, 10 May 2013 23:46:54 -0700 Subject: [PATCH 1/4] Adding an argument to the adapter for passing a block argument to the connection pool. This allows for blocking when using threading to prevent the pool from creating more connections that the max-size allows. Specifically was seeing the following errors without the block=True: WARNING:requests.packages.urllib3.connectionpool:HttpConnectionPool is full, discarding connection: www.example.com --- requests/adapters.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 4245746f..d6952ca4 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -25,6 +25,7 @@ from .cookies import extract_cookies_to_jar from .exceptions import ConnectionError, Timeout, SSLError from .auth import _basic_auth_str +DEFAULT_POOLBLOCK = False DEFAULT_POOLSIZE = 10 DEFAULT_RETRIES = 0 @@ -64,7 +65,8 @@ class HTTPAdapter(BaseAdapter): __attrs__ = ['max_retries', 'config', '_pool_connections', '_pool_maxsize'] def __init__(self, pool_connections=DEFAULT_POOLSIZE, - pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES): + pool_maxsize=DEFAULT_POOLSIZE, pool_block=DEFAULT_POOLBLOCK, + max_retries=DEFAULT_RETRIES): self.max_retries = max_retries self.config = {} @@ -72,8 +74,9 @@ class HTTPAdapter(BaseAdapter): self._pool_connections = pool_connections self._pool_maxsize = pool_maxsize + self._pool_block = pool_block - self.init_poolmanager(pool_connections, pool_maxsize) + self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) def __getstate__(self): return dict((attr, getattr(self, attr, None)) for attr in @@ -85,19 +88,22 @@ class HTTPAdapter(BaseAdapter): self.init_poolmanager(self._pool_connections, self._pool_maxsize) - def init_poolmanager(self, connections, maxsize): + def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK): """Initializes a urllib3 PoolManager. This method should not be called from user code, and is only exposed for use when subclassing the :class:`HTTPAdapter `. :param connections: The number of urllib3 connection pools to cache. :param maxsize: The maximum number of connections to save in the pool. + :param block: Block when no free connections are available. """ # save these values for pickling self._pool_connections = connections self._pool_maxsize = maxsize + self._pool_block = block - self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize) + self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize, + block=block) def cert_verify(self, conn, url, verify, cert): """Verify a SSL certificate. This method should not be called from user From 053613688bcabb05818403014dc594bfdc2cbe22 Mon Sep 17 00:00:00 2001 From: Randy Merrill Date: Sat, 11 May 2013 18:04:43 -0700 Subject: [PATCH 2/4] Moving the order of the arguments in the init to not interfere with existing usage. --- requests/adapters.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index d6952ca4..7d7b822a 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -54,6 +54,7 @@ class HTTPAdapter(BaseAdapter): :param pool_connections: The number of urllib3 connection pools to cache. :param pool_maxsize: The maximum number of connections to save in the pool. :param max_retries: The maximum number of retries each connection should attempt. + :param pool_block: Whether the connection pool should block for connections. Usage:: @@ -65,8 +66,8 @@ class HTTPAdapter(BaseAdapter): __attrs__ = ['max_retries', 'config', '_pool_connections', '_pool_maxsize'] def __init__(self, pool_connections=DEFAULT_POOLSIZE, - pool_maxsize=DEFAULT_POOLSIZE, pool_block=DEFAULT_POOLBLOCK, - max_retries=DEFAULT_RETRIES): + pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES, + pool_block=DEFAULT_POOLBLOCK): self.max_retries = max_retries self.config = {} From 9cb3d6444d7c2e2787267874f59c271cd2bcf1be Mon Sep 17 00:00:00 2001 From: Randy Merrill Date: Sat, 11 May 2013 18:13:08 -0700 Subject: [PATCH 3/4] Fixing the call to init_poolmanagers to correctly unpickle the adapter. --- requests/adapters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requests/adapters.py b/requests/adapters.py index 7d7b822a..97840f99 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -87,7 +87,8 @@ class HTTPAdapter(BaseAdapter): for attr, value in state.items(): setattr(self, attr, value) - self.init_poolmanager(self._pool_connections, self._pool_maxsize) + self.init_poolmanager(self._pool_connections, self._pool_maxsize, + block=self._pool_block) def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK): """Initializes a urllib3 PoolManager. This method should not be called From c03e14242b4bc65f6a970de100460eb3f1e14cc2 Mon Sep 17 00:00:00 2001 From: Randy Merrill Date: Sat, 11 May 2013 18:30:31 -0700 Subject: [PATCH 4/4] Adding the _pool_bloc to the list of attrs. --- requests/adapters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requests/adapters.py b/requests/adapters.py index 97840f99..98b7317e 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -63,7 +63,8 @@ class HTTPAdapter(BaseAdapter): >>> a = requests.adapters.HTTPAdapter() >>> s.mount('http://', a) """ - __attrs__ = ['max_retries', 'config', '_pool_connections', '_pool_maxsize'] + __attrs__ = ['max_retries', 'config', '_pool_connections', '_pool_maxsize', + '_pool_block'] def __init__(self, pool_connections=DEFAULT_POOLSIZE, pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES,