From 8d4afcb5b834abe837a0de74fe11c15006fdef43 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 10 Oct 2011 20:07:10 -0400 Subject: [PATCH] urllib3 upgrade --- requests/packages/urllib3/__init__.py | 24 ++- requests/packages/urllib3/_collections.py | 3 +- requests/packages/urllib3/connectionpool.py | 220 ++++++++++---------- requests/packages/urllib3/filepost.py | 13 ++ requests/packages/urllib3/poolmanager.py | 77 +++++-- requests/packages/urllib3/request.py | 145 +++++++++++++ requests/packages/urllib3/response.py | 19 +- 7 files changed, 357 insertions(+), 144 deletions(-) create mode 100644 requests/packages/urllib3/request.py diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index 5c4b5d17..ae324530 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -8,9 +8,9 @@ urllib3 - Thread-safe connection pooling and re-using. """ -__author__ = "Andrey Petrov (andrey.petrov@shazow.net)" -__license__ = "MIT" -__version__ = "$Rev$" +__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' +__license__ = 'MIT' +__version__ = '1.0' from .connectionpool import ( @@ -27,6 +27,22 @@ from .exceptions import ( SSLError, TimeoutError) -from .poolmanager import PoolManager +from .poolmanager import PoolManager, ProxyManager, proxy_from_url from .response import HTTPResponse from .filepost import encode_multipart_formdata + + +# Set default logging handler to avoid "No handler found" warnings. +import logging +try: + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + +logging.getLogger(__name__).addHandler(NullHandler()) + +# ... Clean up. +del logging +del NullHandler diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py index 2b0de0e8..bd01da33 100644 --- a/requests/packages/urllib3/_collections.py +++ b/requests/packages/urllib3/_collections.py @@ -105,14 +105,13 @@ class RecentlyUsedContainer(MutableMapping): # Discard invalid and excess entries self._prune_entries(len(self._container) - self._maxsize) - def __delitem__(self, key): self._invalidate_entry(key) del self._container[key] del self._access_lookup[key] def __len__(self): - return len(self.access_log) + return self._container.__len__() def __iter__(self): return self._container.__iter__() diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 8f74b1f5..7f974dfc 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -8,7 +8,6 @@ import logging import socket -from urllib import urlencode from httplib import HTTPConnection, HTTPSConnection, HTTPException from Queue import Queue, Empty, Full from select import select @@ -23,14 +22,15 @@ except ImportError: BaseSSLError = None -from .filepost import encode_multipart_formdata +from .request import RequestMethods from .response import HTTPResponse from .exceptions import ( SSLError, MaxRetryError, TimeoutError, HostChangedError, - EmptyPoolError) + EmptyPoolError, +) log = logging.getLogger(__name__) @@ -38,7 +38,6 @@ log = logging.getLogger(__name__) _Default = object() - ## Connection objects (extension of httplib) class VerifiedHTTPSConnection(HTTPSConnection): @@ -46,11 +45,8 @@ class VerifiedHTTPSConnection(HTTPSConnection): Based on httplib.HTTPSConnection but wraps the socket with SSL certification. """ - - def __init__(self, **kwargs): - HTTPSConnection.__init__(self, **kwargs) - self.cert_reqs = None - self.ca_certs = None + cert_reqs = None + cert_reqs = None def set_cert(self, key_file=None, cert_file=None, cert_reqs='CERT_NONE', ca_certs=None): @@ -79,44 +75,48 @@ class VerifiedHTTPSConnection(HTTPSConnection): ## Pool objects class ConnectionPool(object): + """ + Base class for all connection pools, such as + :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`. + """ pass -class HTTPConnectionPool(ConnectionPool): +class HTTPConnectionPool(ConnectionPool, RequestMethods): """ Thread-safe connection pool for one host. - host + :param host: Host used for this HTTP Connection (e.g. "localhost"), passed into - httplib.HTTPConnection() + :class:`httplib.HTTPConnection`. - port + :param port: Port used for this HTTP Connection (None is equivalent to 80), passed - into httplib.HTTPConnection() + into :class:`httplib.HTTPConnection`. - strict + :param strict: Causes BadStatusLine to be raised if the status line can't be parsed as a valid HTTP/1.0 or 1.1 status line, passed into - httplib.HTTPConnection() + :class:`httplib.HTTPConnection`. - timeout + :param timeout: Socket timeout for each individual connection, can be a float. None disables timeout. - maxsize + :param maxsize: Number of connections to save that can be reused. More than 1 is useful in multithreaded situations. If ``block`` is set to false, more connections will be created but they will not be saved once they've been used. - block + :param block: If set to True, no more than ``maxsize`` connections will be used at a time. When no free connections are available, the call will block until a connection has been released. This is a useful side effect for particular multithreaded situations where one does not want to use more than maxsize connections per host to prevent flooding. - headers + :param headers: Headers to include with all requests, unless other headers are given explicitly. """ @@ -143,7 +143,7 @@ class HTTPConnectionPool(ConnectionPool): def _new_conn(self): """ - Return a fresh HTTPConnection. + Return a fresh :class:`httplib.HTTPConnection`. """ self.num_connections += 1 log.info("Starting new HTTP connection (%d): %s" % @@ -153,7 +153,14 @@ class HTTPConnectionPool(ConnectionPool): def _get_conn(self, timeout=None): """ Get a connection. Will return a pooled connection if one is available. - Otherwise, a fresh connection is returned. + + If no connections are available and :prop:`.block` is ``False``, then a + fresh connection is returned. + + :param timeout: + Seconds to wait before giving up and raising + :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and + :prop:`.block` is ``True``. """ conn = None try: @@ -176,6 +183,11 @@ class HTTPConnectionPool(ConnectionPool): def _put_conn(self, conn): """ Put a connection back into the pool. + + :param conn: + Connection object for the current host and port as returned by + :meth:`._new_conn` or :meth:`._get_conn`. + If the pool is already full, the connection is discarded because we exceeded maxsize. If connections are discarded frequently, then maxsize should be increased. @@ -198,12 +210,10 @@ class HTTPConnectionPool(ConnectionPool): if timeout is _Default: timeout = self.timeout - conn.request(method, url, **httplib_request_kw) conn.sock.settimeout(timeout) httplib_response = conn.getresponse() - log.debug("\"%s %s %s\" %s %s" % (method, url, conn._http_vsn_str, # pylint: disable-msg=W0212 @@ -213,6 +223,11 @@ class HTTPConnectionPool(ConnectionPool): def is_same_host(self, url): + """ + Check if the given ``url`` is a member of the same host as this + conncetion pool. + """ + # TODO: Add optional support for socket.gethostbyname checking. return (url.startswith('/') or get_host(url) == (self.scheme, self.host, self.port)) @@ -220,43 +235,50 @@ class HTTPConnectionPool(ConnectionPool): redirect=True, assert_same_host=True, timeout=_Default, pool_timeout=None, release_conn=None, **response_kw): """ - Get a connection from the pool and perform an HTTP request. + Get a connection from the pool and perform an HTTP request. This is the + lowest level call for making a request, so you'll need to specify all + the raw details. - method + .. note:: + + More commonly, it's appropriate to use a convenience method provided + by :class:`.RequestMethods`, such as :meth:`.request`. + + :param method: HTTP request method (such as GET, POST, PUT, etc.) - body + :param body: Data to send in the request body (useful for creating POST requests, see HTTPConnectionPool.post_url for more convenience). - headers + :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. - retries + :param retries: Number of retries to allow before raising a MaxRetryError exception. - redirect + :param redirect: Automatically handle redirects (status codes 301, 302, 303, 307), each redirect counts as a retry. - assert_same_host - If True, will make sure that the host of the pool requests is + :param assert_same_host: + If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When False, you can use the pool on an HTTP proxy and request foreign hosts. - timeout + :param timeout: If specified, overrides the default timeout for this one request. - pool_timeout + :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. - release_conn + :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received. This is useful if you're not preloading the response's content immediately. You will @@ -264,8 +286,9 @@ class HTTPConnectionPool(ConnectionPool): the connection back into the pool. If None, it takes the value of ``response_kw.get('preload_content', True)``. - Additional parameters are passed to - ``HTTPResponse.from_httplib(r, **response_kw)`` + :param \**response_kw: + Additional parameters are passed to + :meth:`urllib3.response.HTTPResponse.from_httplib` """ if headers is None: headers = self.headers @@ -291,11 +314,11 @@ class HTTPConnectionPool(ConnectionPool): # Request a connection from the queue # (Could raise SocketError: Bad file descriptor) conn = self._get_conn(timeout=pool_timeout) + # Make the request on the httplib connection object httplib_response = self._make_request(conn, method, url, timeout=timeout, body=body, headers=headers) - # print '!' # Import httplib's response into our own wrapper object response = HTTPResponse.from_httplib(httplib_response, @@ -348,66 +371,18 @@ class HTTPConnectionPool(ConnectionPool): return response - def get_url(self, url, fields=None, headers=None, retries=3, - redirect=True, **response_kw): - """ - Wrapper for performing GET with urlopen (see urlopen for more details). - - Supports an optional ``fields`` dictionary parameter key/value strings. - If provided, they will be added to the url. - """ - if fields: - url += '?' + urlencode(fields) - return self.urlopen('GET', url, headers=headers, retries=retries, - redirect=redirect, **response_kw) - - def post_url(self, url, fields=None, headers=None, retries=3, - redirect=True, encode_multipart=True, multipart_boundary=None, - **response_kw): - """ - Wrapper for performing POST with urlopen (see urlopen - for more details). - - Supports an optional ``fields`` parameter of key/value strings AND - key/filetuple. A filetuple is a (filename, data) tuple. For example: - - fields = { - 'foo': 'bar', - 'foofile': ('foofile.txt', 'contents of foofile'), - } - - If encode_multipart=True (default), then - ``urllib3.filepost.encode_multipart_formdata`` is used to encode the - payload with the appropriate content type. Otherwise - ``urllib.urlencode`` is used with 'application/x-www-form-urlencoded' - content type. - - Multipart encoding must be used when posting files, and it's reasonably - safe to use it other times too. It may break request signing, such as - OAuth. - - NOTE: If ``headers`` are supplied, the 'Content-Type' value will be - overwritten because it depends on the dynamic random boundary string - which is used to compose the body of the request. - """ - if encode_multipart: - body, content_type = encode_multipart_formdata(fields or {}, - boundary=multipart_boundary) - else: - body, content_type = ( - urlencode(fields or {}), - 'application/x-www-form-urlencoded') - - headers = headers or {} - headers.update({'Content-Type': content_type}) - - return self.urlopen('POST', url, body, headers=headers, - retries=retries, redirect=redirect, **response_kw) - class HTTPSConnectionPool(HTTPConnectionPool): """ - Same as HTTPConnectionPool, but HTTPS. + Same as :class:`.HTTPConnectionPool`, but HTTPS. + + When Python is compiled with the :mod:`ssl` module, then + :class:`.VerifiedHTTPSConnection` is used, which *can* verify certificates, + instead of :class:httplib.HTTPSConnection`. + + The ``key_file``, ``cert_file``, ``cert_reqs``, and ``ca_certs`` parameters + are only used if :mod:`ssl` is available and are fed into + :meth:`ssl.wrap_socket` to upgrade the connection socket into an SSL socket. """ scheme = 'https' @@ -416,7 +391,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): strict=False, timeout=None, maxsize=1, block=False, headers=None, key_file=None, cert_file=None, - cert_reqs=ssl.CERT_REQUIRED, ca_certs=None): + cert_reqs='CERT_NONE', ca_certs=None): super(HTTPSConnectionPool, self).__init__(host, port, strict, timeout, maxsize, @@ -428,9 +403,8 @@ class HTTPSConnectionPool(HTTPConnectionPool): def _new_conn(self): """ - Return a fresh HTTPSConnection. + Return a fresh :class:`httplib.HTTPSConnection`. """ - self.num_connections += 1 log.info("Starting new HTTPS connection (%d): %s" % (self.num_connections, self.host)) @@ -451,22 +425,29 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, """ Shortcuts for generating request headers. - keep_alive - If true, adds 'connection: keep-alive' header. + :param keep_alive: + If ``True``, adds 'connection: keep-alive' header. - accept_encoding + :param accept_encoding: Can be a boolean, list, or string. - True translates to 'gzip,deflate'. + ``True`` translates to 'gzip,deflate'. List will get joined by comma. String will be used as provided. - user_agent + :param user_agent: String representing the user-agent you want, such as "python-urllib3/0.6" - basic_auth + :param basic_auth: Colon-separated username:password string for 'authorization: basic ...' auth header. + + Example: :: + + >>> make_headers(keep_alive=True, user_agent="Batman/1.0") + {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} + >>> make_headers(accept_encoding=True) + {'accept-encoding': 'gzip,deflate'} """ headers = {} if accept_encoding: @@ -495,11 +476,12 @@ def get_host(url): """ Given a url, return its scheme, host and port (None if it's not there). - For example: - >>> get_host('http://google.com/mail/') - http, google.com, None - >>> get_host('google.com:80') - http, google.com, 80 + For example: :: + + >>> get_host('http://google.com/mail/') + ('http', 'google.com', None) + >>> get_host('google.com:80') + ('http', 'google.com', 80) """ # This code is actually similar to urlparse.urlsplit, but much # simplified for our needs. @@ -517,13 +499,23 @@ def get_host(url): def connection_from_url(url, **kw): """ - Given a url, return an HTTP(S)ConnectionPool instance of its host. + Given a url, return an :class:`.ConnectionPool` instance of its host. - This is a shortcut for not having to determine the host of the url - before creating an HTTP(S)ConnectionPool instance. + This is a shortcut for not having to parse out the scheme, host, and port + of the url before creating an :class:`.ConnectionPool` instance. - Passes on whatever kw arguments to the constructor of - HTTP(S)ConnectionPool. (e.g. timeout, maxsize, block) + :param url: + Absolute URL string that must include the scheme. Port is optional. + + :param \**kw: + Passes additional parameters to the constructor of the appropriate + :class:`.ConnectionPool`. Useful for specifying things like + timeout, maxsize, headers, etc. + + Example: :: + + >>> conn = connection_from_url('http://google.com/') + >>> r = conn.request('GET', '/') """ scheme, host, port = get_host(url) if scheme == 'https': diff --git a/requests/packages/urllib3/filepost.py b/requests/packages/urllib3/filepost.py index 8e65b5c2..2ffea8bb 100644 --- a/requests/packages/urllib3/filepost.py +++ b/requests/packages/urllib3/filepost.py @@ -22,6 +22,19 @@ def get_content_type(filename): def encode_multipart_formdata(fields, boundary=None): + """ + Encode a dictionary of ``fields`` using the multipart/form-data mime format. + + :param fields: + Dictionary of fields. The key is treated as the field name, and the + value as the body of the form-data. If the value is a tuple of two + elements, then the first element is treated as the filename of the + form-data section. + + :param boundary: + If not specified, then a random boundary will be generated using + :func:`mimetools.choose_boundary`. + """ body = StringIO() if boundary is None: boundary = mimetools.choose_boundary() diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index 1d757d19..622789e5 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -5,7 +5,17 @@ # the MIT License: http://www.opensource.org/licenses/mit-license.php from ._collections import RecentlyUsedContainer -from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, get_host +from .connectionpool import ( + HTTPConnectionPool, HTTPSConnectionPool, + get_host, connection_from_url, +) + + +__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] + + +from .request import RequestMethods +from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool pool_classes_by_scheme = { @@ -19,16 +29,27 @@ port_by_scheme = { } -class PoolManager(object): +class PoolManager(RequestMethods): """ Allows for arbitrary requests while transparently keeping track of necessary connection pools for you. - num_pools + :param num_pools: Number of connection pools to cache before discarding the least recently used pool. - Additional parameters are used to create fresh ConnectionPool instances. + :param \**connection_pool_kw: + Additional parameters are used to create fresh + :class:`urllib3.connectionpool.ConnectionPool` instances. + + Example: :: + + >>> manager = PoolManager() + >>> r = manager.urlopen("http://google.com/") + >>> r = manager.urlopen("http://google.com/mail") + >>> r = manager.urlopen("http://yahoo.com/") + >>> len(r.pools) + 2 """ @@ -36,17 +57,17 @@ class PoolManager(object): def __init__(self, num_pools=10, **connection_pool_kw): self.connection_pool_kw = connection_pool_kw - self.pools = RecentlyUsedContainer(num_pools) - self.recently_used_pools = [] def connection_from_host(self, host, port=80, scheme='http'): """ - Get a ConnectionPool based on the host, port, and scheme. + Get a :class:`ConnectionPool` based on the host, port, and scheme. + + Note that an appropriate ``port`` value is required here to normalize + connection pools in our container most effectively. """ pool_key = (scheme, host, port) - # If the scheme, host, or port doesn't match existing open connections, # open a new ConnectionPool. pool = self.pools.get(pool_key) @@ -63,19 +84,45 @@ class PoolManager(object): def connection_from_url(self, url): """ - Similar to connectionpool.connection_from_url but doesn't pass any - additional keywords to the ConnectionPool constructor. Additional - keywords are taken from the PoolManager constructor. + Similar to :func:`urllib3.connectionpool.connection_from_url` but + doesn't pass any additional parameters to the + :class:`urllib3.connectionpool.ConnectionPool` constructor. + + Additional parameters are taken from the :class:`.PoolManager` + constructor. """ scheme, host, port = get_host(url) port = port or port_by_scheme.get(scheme, 80) - r = self.connection_from_host(host, port=port, scheme=scheme) + return self.connection_from_host(host, port=port, scheme=scheme) - return r + def urlopen(self, method, url, **kw): + """ + Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`. + + ``url`` must be absolute, such that an appropriate + :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it. + """ + conn = self.connection_from_url(url) + return conn.urlopen(method, url, assert_same_host=False, **kw) + + +class ProxyManager(object): + """ + Given a ConnectionPool to a proxy, the ProxyManager's ``urlopen`` method + will make requests to any url through the defined proxy. + """ + + def __init__(self, proxy_pool): + self.proxy_pool = proxy_pool def urlopen(self, method, url, **kw): "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." - conn = self.connection_from_url(url) - return conn.urlopen(method, url, **kw) + kw['assert_same_host'] = False + return self.proxy_pool.urlopen(method, url, **kw) + + +def proxy_from_url(url, **pool_kw): + proxy_pool = connection_from_url(url, **pool_kw) + return ProxyManager(proxy_pool) diff --git a/requests/packages/urllib3/request.py b/requests/packages/urllib3/request.py new file mode 100644 index 00000000..a7e0b5de --- /dev/null +++ b/requests/packages/urllib3/request.py @@ -0,0 +1,145 @@ +# urllib3/request.py +# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + + +from urllib import urlencode + +from .filepost import encode_multipart_formdata + + +__all__ = ['RequestMethods'] + + +class RequestMethods(object): + """ + Convenience mixin for classes who implement a :meth:`urlopen` method, such + as :class:`~urllib3.connectionpool.HTTPConnectionPool` and + :class:`~urllib3.poolmanager.PoolManager`. + + Provides behavior for making common types of HTTP request methods and + decides which type of request field encoding to use. + + Specifically, + + :meth:`.request_encode_url` is for sending requests whose fields are encoded + in the URL (such as GET, HEAD, DELETE). + + :meth:`.request_encode_body` is for sending requests whose fields are + encoded in the *body* of the request using multipart or www-orm-urlencoded + (such as for POST, PUT, PATCH). + + :meth:`.request` is for making any kind of request, it will look up the + appropriate encoding format and use one of the above two methods to make + the request. + """ + + _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS']) + + _encode_body_methods = set(['PATCH', 'POST', 'PUT', 'TRACE']) + + def urlopen(self, method, url, body=None, headers=None, + encode_multipart=True, multipart_boundary=None, + **kw): + raise NotImplemented("Classes extending RequestMethods must implement " + "their own ``urlopen`` method.") + + def request(self, method, url, fields=None, headers=None, **urlopen_kw): + """ + Make a request using :meth:`urlopen` with the appropriate encoding of + ``fields`` based on the ``method`` used. + + This is a convenience method that requires the least amount of manual + effort. It can be used in most situations, while still having the option + to drop down to more specific methods when necessary, such as + :meth:`request_encode_url`, :meth:`request_encode_body`, + or even the lowest level :meth:`urlopen`. + """ + method = method.upper() + + if method in self._encode_url_methods: + return self.request_encode_url(method, url, fields=fields, + headers=headers, + **urlopen_kw) + else: + return self.request_encode_body(method, url, fields=fields, + headers=headers, + **urlopen_kw) + + def request_encode_url(self, method, url, fields=None, **urlopen_kw): + """ + Make a request using :meth:`urlopen` with the ``fields`` encoded in + the url. This is useful for request methods like GET, HEAD, DELETE, etc. + """ + if fields: + url += '?' + urlencode(fields) + return self.urlopen(method, url, **urlopen_kw) + + def request_encode_body(self, method, url, fields=None, headers=None, + encode_multipart=True, multipart_boundary=None, + **urlopen_kw): + """ + Make a request using :meth:`urlopen` with the ``fields`` encoded in + the body. This is useful for request methods like POST, PUT, PATCH, etc. + + When ``encode_multipart=True`` (default), then + :meth:`urllib3.filepost.encode_multipart_formdata` is used to encode the + payload with the appropriate content type. Otherwise + :meth:`urllib.urlencode` is used with the + 'application/x-www-form-urlencoded' content type. + + Multipart encoding must be used when posting files, and it's reasonably + safe to use it in other times too. However, it may break request signing, + such as with OAuth. + + Supports an optional ``fields`` parameter of key/value strings AND + key/filetuple. A filetuple is a (filename, data) tuple. For example: :: + + fields = { + 'foo': 'bar', + 'fakefile': ('foofile.txt', 'contents of foofile'), + 'realfile': ('barfile.txt', open('realfile').read()), + 'nonamefile': ('contents of nonamefile field'), + } + + When uploading a file, providing a filename (the first parameter of the + tuple) is optional but recommended to best mimick behavior of browsers. + + Note that if ``headers`` are supplied, the 'Content-Type' header will be + overwritten because it depends on the dynamic random boundary string + which is used to compose the body of the request. The random boundary + string can be explicitly set with the ``multipart_boundary`` parameter. + """ + if encode_multipart: + body, content_type = encode_multipart_formdata(fields or {}, + boundary=multipart_boundary) + else: + body, content_type = (urlencode(fields or {}), + 'application/x-www-form-urlencoded') + + headers = headers or {} + headers.update({'Content-Type': content_type}) + + return self.urlopen(method, url, body=body, headers=headers, + **urlopen_kw) + + # Deprecated: + + def get_url(self, url, fields=None, **urlopen_kw): + """ + .. deprecated:: 1.0 + Use :meth:`request` instead. + """ + return self.request_encode_url('GET', url, fields=fields, + **urlopen_kw) + + def post_url(self, url, fields=None, headers=None, **urlopen_kw): + """ + .. deprecated:: 1.0 + Use :meth:`request` instead. + """ + return self.request_encode_body('POST', url, fields=fields, + headers=headers, + **urlopen_kw) diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 2bbc06ab..4cd15c11 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -42,15 +42,15 @@ class HTTPResponse(object): Extra parameters for behaviour not present in httplib.HTTPResponse: - preload_content + :param preload_content: If True, the response's body will be preloaded during construction. - decode_content + :param decode_content: If True, attempts to decode specific content-encoding's based on headers (like 'gzip' and 'deflate') will be skipped and raw data will be used instead. - original_response + :param original_response: When this HTTPResponse wrapper is generated from an httplib.HTTPResponse object, it's convenient to include the original for debug purposes. It's otherwise unused. @@ -103,18 +103,19 @@ class HTTPResponse(object): def read(self, amt=None, decode_content=True, cache_content=False): """ - Similar to ``httplib.HTTPResponse.read(amt=None)``. + Similar to :meth:`httplib.HTTPResponse.read`, but with two additional + parameters: ``decode_content`` and ``cache_content``. - amt + :param amt: How much of the content to read. If specified, decoding and caching is skipped because we can't decode partial content nor does it make sense to cache partial content as the full response. - decode_content + :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. (Overridden if ``amt`` is set.) - cache_content + :param cache_content: If True, will save the returned data such that the same result is returned despite of the state of the underlying file object. This is useful if you want the ``.data`` property to continue working @@ -156,8 +157,8 @@ class HTTPResponse(object): @staticmethod def from_httplib(r, **response_kw): """ - Given an httplib.HTTPResponse instance ``r``, return a corresponding - urllib3.HTTPResponse object. + Given an :class:`httplib.HTTPResponse` instance ``r``, return a + corresponding :class:`urllib3.response.HTTPResponse` object. Remaining parameters are passed to the HTTPResponse constructor, along with ``original_response=r``.