Merge remote-tracking branch 'origin/master'

This commit is contained in:
Kenneth Reitz
2013-07-16 02:20:28 -04:00
22 changed files with 269 additions and 70 deletions
+4
View File
@@ -125,6 +125,10 @@ Patches and Suggestions
- Dmitry Medvinsky <me@dmedvinsky.name>
- Bryce Boe <bbzbryce@gmail.com> @bboe
- Colin Dunklau <colin.dunklau@gmail.com> @cdunklau
- Bob Carroll <bob.carroll@alum.rit.edu> @rcarz
- Hugo Osvaldo Barrera <hugo@osvaldobarrera.com.ar> @hobarrera
- Łukasz Langa <lukasz@langa.pl> @llanga
- Dave Shawley <daveshawley@gmail.com>
- James Clarke (jam)
- Kevin Burke <kev@inburke.com>
- Flavio Curella
+1 -2
View File
@@ -11,8 +11,7 @@
<p>
Requests is an elegant and simple HTTP library for Python, built for
human beings. You are currently looking at the documentation of the
development release.
human beings.
</p>
+5
View File
@@ -53,3 +53,8 @@ Are you crazy?
--------------
- SPDY support would be awesome. No C extensions.
Downstream Repackaging
--------------------
If you are repackaging Requests, please note that you must also redistribute the ``cacerts.pem`` file in order to get correct SSL functionality.
+20 -2
View File
@@ -49,7 +49,7 @@ Request and Response Objects
----------------------------
Whenever a call is made to requests.*() you are doing two major things. First,
you are constructing a ``Request`` object which will be sent of to a server
you are constructing a ``Request`` object which will be sent off to a server
to request or query some resource. Second, a ``Response`` object is generated
once ``requests`` gets a response back from the server. The response object
contains all of the information returned by the server and also contains the
@@ -268,6 +268,8 @@ Then, we can make a request using our Pizza Auth::
>>> requests.get('http://pizzabin.org/admin', auth=PizzaAuth('kenneth'))
<Response [200]>
.. _streaming-requests
Streaming Requests
------------------
@@ -279,7 +281,7 @@ To use the Twitter Streaming API to track the keyword "requests"::
import json
import requests
r = requests.post('http://httpbin.org/stream/20', stream=True)
r = requests.get('http://httpbin.org/stream/20', stream=True)
for line in r.iter_lines():
@@ -574,3 +576,19 @@ a good start would be to subclass the ``requests.adapters.BaseAdapter`` class.
.. _`described here`: http://kennethreitz.org/exposures/the-future-of-python-http
.. _`urllib3`: https://github.com/shazow/urllib3
Blocking Or Non-Blocking?
-------------------------
With the default Transport Adapter in place, Requests does not provide any kind
of non-blocking IO. The ``Response.content`` property will block until the
entire response has been downloaded. If you require more granularity, the
streaming features of the library (see :ref:`streaming-requests`) allow you to
retrieve smaller quantities of the response at a time. However, these calls
will still block.
If you are concerned about the use of blocking IO, there are lots of projects
out there that combine Requests with one of Python's asynchronicity frameworks.
Two excellent examples are `grequests`_ and `requests-futures`_.
.. _`grequests`: https://github.com/kennethreitz/grequests
.. _`requests-futures`: https://github.com/ross/requests-futures
+7 -6
View File
@@ -260,7 +260,7 @@ reference::
>>> r.status_code == requests.codes.ok
True
If we made a bad request (non-200 response), we can raise it with
If we made a bad request (a 4XX client error or 5XX server error response), we can raise it with
:class:`Response.raise_for_status()`::
>>> bad_r = requests.get('http://httpbin.org/status/404')
@@ -399,15 +399,16 @@ Errors and Exceptions
---------------------
In the event of a network problem (e.g. DNS failure, refused connection, etc),
Requests will raise a :class:`ConnectionError` exception.
Requests will raise a :class:`~requests.exceptions.ConnectionError` exception.
In the event of the rare invalid HTTP response, Requests will raise
an :class:`HTTPError` exception.
In the event of the rare invalid HTTP response, Requests will raise an
:class:`~requests.exceptions.HTTPError` exception.
If a request times out, a :class:`Timeout` exception is raised.
If a request times out, a :class:`~requests.exceptions.Timeout` exception is
raised.
If a request exceeds the configured number of maximum redirections, a
:class:`TooManyRedirects` exception is raised.
:class:`~requests.exceptions.TooManyRedirects` exception is raised.
All exceptions that Requests explicitly raises inherit from
:class:`requests.exceptions.RequestException`.
-5
View File
@@ -1,5 +0,0 @@
from invoke import run, task
@task
def build():
print("Building!")
+4 -4
View File
@@ -118,7 +118,7 @@ class HTTPAdapter(BaseAdapter):
:param verify: Whether we should actually verify the certificate.
:param cert: The SSL certificate to verify.
"""
if url.startswith('https') and verify:
if url.lower().startswith('https') and verify:
cert_loc = None
@@ -190,13 +190,13 @@ class HTTPAdapter(BaseAdapter):
:param proxies: (optional) A Requests-style dictionary of proxies used on this request.
"""
proxies = proxies or {}
proxy = proxies.get(urlparse(url).scheme)
proxy = proxies.get(urlparse(url.lower()).scheme)
if proxy:
proxy = prepend_scheme_if_needed(proxy, urlparse(url).scheme)
proxy = prepend_scheme_if_needed(proxy, urlparse(url.lower()).scheme)
conn = ProxyManager(self.poolmanager.connection_from_url(proxy))
else:
conn = self.poolmanager.connection_from_url(url)
conn = self.poolmanager.connection_from_url(url.lower())
return conn
+2 -2
View File
@@ -83,7 +83,7 @@ except ImportError:
# ---------
if is_py2:
from urllib import quote, unquote, quote_plus, unquote_plus, urlencode
from urllib import quote, unquote, quote_plus, unquote_plus, urlencode, getproxies, proxy_bypass
from urlparse import urlparse, urlunparse, urljoin, urlsplit, urldefrag
from urllib2 import parse_http_list
import cookielib
@@ -100,7 +100,7 @@ if is_py2:
elif is_py3:
from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, quote_plus, unquote_plus, urldefrag
from urllib.request import parse_http_list
from urllib.request import parse_http_list, getproxies, proxy_bypass
from http import cookiejar as cookielib
from http.cookies import Morsel
from io import StringIO
+15 -5
View File
@@ -6,6 +6,7 @@ Compatibility code to be able to use `cookielib.CookieJar` with requests.
requests.utils imports from here, so be careful with imports.
"""
import time
import collections
from .compat import cookielib, urlparse, Morsel
@@ -258,6 +259,11 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping):
"""Deletes a cookie given a name. Wraps cookielib.CookieJar's remove_cookie_by_name()."""
remove_cookie_by_name(self, name)
def set_cookie(self, cookie, *args, **kwargs):
if cookie.value.startswith('"') and cookie.value.endswith('"'):
cookie.value = cookie.value.replace('\\"', '')
return super(RequestsCookieJar, self).set_cookie(cookie, *args, **kwargs)
def update(self, other):
"""Updates this jar with cookies from another CookieJar or dict-like"""
if isinstance(other, cookielib.CookieJar):
@@ -354,19 +360,23 @@ def create_cookie(name, value, **kwargs):
def morsel_to_cookie(morsel):
"""Convert a Morsel object into a Cookie containing the one k/v pair."""
expires = None
if morsel["max-age"]:
expires = time.time() + morsel["max-age"]
elif morsel['expires']:
expires = morsel['expires']
if type(expires) == type(""):
time_template = "%a, %d-%b-%Y %H:%M:%S GMT"
expires = time.mktime(time.strptime(expires, time_template))
c = create_cookie(
name=morsel.key,
value=morsel.value,
version=morsel['version'] or 0,
port=None,
port_specified=False,
domain=morsel['domain'],
domain_specified=bool(morsel['domain']),
domain_initial_dot=morsel['domain'].startswith('.'),
path=morsel['path'],
path_specified=bool(morsel['path']),
secure=bool(morsel['secure']),
expires=morsel['max-age'] or morsel['expires'],
expires=expires,
discard=False,
comment=morsel['comment'],
comment_url=bool(morsel['comment']),
+14 -7
View File
@@ -364,7 +364,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
try:
length = super_len(data)
except (TypeError, AttributeError):
length = False
length = None
if is_stream:
body = data
@@ -372,7 +372,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
if files:
raise NotImplementedError('Streamed bodies and files are mutually exclusive.')
if length:
if length is not None:
self.headers['Content-Length'] = str(length)
else:
self.headers['Transfer-Encoding'] = 'chunked'
@@ -537,11 +537,18 @@ class Response(object):
return iter_slices(self._content, chunk_size)
def generate():
while 1:
chunk = self.raw.read(chunk_size, decode_content=True)
if not chunk:
break
yield chunk
try:
# Special case for urllib3.
for chunk in self.raw.stream(chunk_size, decode_content=True):
yield chunk
except AttributeError:
# Standard file-like object.
while 1:
chunk = self.raw.read(chunk_size)
if not chunk:
break
yield chunk
self._content_consumed = True
gen = generate()
+2 -1
View File
@@ -110,7 +110,7 @@ class VerifiedHTTPSConnection(HTTPSConnection):
if self.assert_fingerprint:
assert_fingerprint(self.sock.getpeercert(binary_form=True),
self.assert_fingerprint)
else:
elif self.assert_hostname is not False:
match_hostname(self.sock.getpeercert(),
self.assert_hostname or self.host)
@@ -513,6 +513,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
:class:`.VerifiedHTTPSConnection` uses one of ``assert_fingerprint``,
``assert_hostname`` and ``host`` in this order to verify connections.
If ``assert_hostname`` is False, no verification is done.
The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs`` and
``ssl_version`` are only used if :mod:`ssl` is available and are fed into
@@ -33,7 +33,7 @@ class NTLMConnectionPool(HTTPSConnectionPool):
def __init__(self, user, pw, authurl, *args, **kwargs):
"""
authurl is a random URL on the server that is protected by NTLM.
user is the Windows user, probably in the DOMAIN\username format.
user is the Windows user, probably in the DOMAIN\\username format.
pw is the password for the user.
"""
super(NTLMConnectionPool, self).__init__(*args, **kwargs)
@@ -115,6 +115,9 @@ class WrappedSocket(object):
def sendall(self, data):
return self.connection.sendall(data)
def close(self):
return self.connection.shutdown()
def getpeercert(self, binary_form=False):
x509 = self.connection.get_peer_certificate()
if not x509:
+1 -1
View File
@@ -1,5 +1,5 @@
# urllib3/filepost.py
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
+9
View File
@@ -6,6 +6,11 @@
import logging
try: # Python 3
from urllib.parse import urljoin
except ImportError:
from urlparse import urljoin
from ._collections import RecentlyUsedContainer
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
from .connectionpool import connection_from_url, port_by_scheme
@@ -145,6 +150,10 @@ class PoolManager(RequestMethods):
if not redirect_location:
return response
# Support relative URLs for redirecting.
redirect_location = urljoin(url, redirect_location)
# RFC 2616, Section 10.3.4
if response.status == 303:
method = 'GET'
+1 -1
View File
@@ -30,7 +30,7 @@ class RequestMethods(object):
in the URL (such as GET, HEAD, DELETE).
:meth:`.request_encode_body` is for sending requests whose fields are
encoded in the *body* of the request using multipart or www-orm-urlencoded
encoded in the *body* of the request using multipart or www-form-urlencoded
(such as for POST, PUT, PATCH).
:meth:`.request` is for making any kind of request, it will look up the
+59 -2
View File
@@ -1,5 +1,5 @@
# urllib3/response.py
# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
#
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -7,9 +7,11 @@
import logging
import zlib
import io
from .exceptions import DecodeError
from .packages.six import string_types as basestring, binary_type
from .util import is_fp_closed
log = logging.getLogger(__name__)
@@ -48,7 +50,7 @@ def _get_decoder(mode):
return DeflateDecoder()
class HTTPResponse(object):
class HTTPResponse(io.IOBase):
"""
HTTP Response container.
@@ -200,6 +202,29 @@ class HTTPResponse(object):
if self._original_response and self._original_response.isclosed():
self.release_conn()
def stream(self, amt=2**16, decode_content=None):
"""
A generator wrapper for the read() method. A call will block until
``amt`` bytes have been read from the connection or until the
connection is closed.
:param amt:
How much of the content to read. The generator will return up to
much data per iteration, but may return less. This is particularly
likely when using compressed data. However, the empty string will
never be returned.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
"""
while not is_fp_closed(self._fp):
data = self.read(amt=amt, decode_content=decode_content)
if data:
yield data
@classmethod
def from_httplib(ResponseCls, r, **response_kw):
"""
@@ -239,3 +264,35 @@ class HTTPResponse(object):
def getheader(self, name, default=None):
return self.headers.get(name, default)
# Overrides from io.IOBase
def close(self):
if not self.closed:
self._fp.close()
@property
def closed(self):
if self._fp is None:
return True
elif hasattr(self._fp, 'closed'):
return self._fp.closed
elif hasattr(self._fp, 'isclosed'): # Python 2
return self._fp.isclosed()
else:
return True
def fileno(self):
if self._fp is None:
raise IOError("HTTPResponse has no file to get a fileno from")
elif hasattr(self._fp, "fileno"):
return self._fp.fileno()
else:
raise IOError("The file-like object this HTTPResponse is wrapped "
"around has no file descriptor")
def flush(self):
if self._fp is not None and hasattr(self._fp, 'flush'):
return self._fp.flush()
def readable(self):
return True
+14 -1
View File
@@ -31,7 +31,6 @@ try: # Test for SSL features
except ImportError:
pass
from .packages import six
from .exceptions import LocationParseError, SSLError
@@ -341,6 +340,20 @@ def assert_fingerprint(cert, fingerprint):
.format(hexlify(fingerprint_bytes),
hexlify(cert_digest)))
def is_fp_closed(obj):
"""
Checks whether a given file-like object is closed.
:param obj:
The file-like object to check.
"""
if hasattr(obj, 'fp'):
# Object is a container for another file-like object that gets released
# on exhaustion (e.g. HTTPResponse)
return obj.fp is None
return obj.closed
if SSLContext is not None: # Python 3.2+
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
+15 -12
View File
@@ -71,15 +71,13 @@ class SessionRedirectMixin(object):
"""Receives a Response. Returns a generator of Responses."""
i = 0
prepared_request = PreparedRequest()
prepared_request.body = req.body
prepared_request.headers = req.headers.copy()
prepared_request.hooks = req.hooks
prepared_request.method = req.method
prepared_request.url = req.url
# ((resp.status_code is codes.see_other))
while (('location' in resp.headers and resp.status_code in REDIRECT_STATI)):
prepared_request = PreparedRequest()
prepared_request.body = req.body
prepared_request.headers = req.headers.copy()
prepared_request.hooks = req.hooks
resp.content # Consume socket so it can be released
@@ -90,13 +88,18 @@ class SessionRedirectMixin(object):
resp.close()
url = resp.headers['location']
method = prepared_request.method
method = req.method
# Handle redirection without scheme (see: RFC 1808 Section 4)
if url.startswith('//'):
parsed_rurl = urlparse(resp.url)
url = '%s:%s' % (parsed_rurl.scheme, url)
# The scheme should be lower case...
if '://' in url:
scheme, uri = url.split('://', 1)
url = '%s://%s' % (scheme.lower(), uri)
# Facilitate non-RFC2616-compliant 'location' headers
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
# Compliant with RFC3986, we percent encode the url.
@@ -109,12 +112,12 @@ class SessionRedirectMixin(object):
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
if (resp.status_code == codes.see_other and
prepared_request.method != 'HEAD'):
method != 'HEAD'):
method = 'GET'
# Do what the browsers do, despite standards...
if (resp.status_code in (codes.moved, codes.found) and
prepared_request.method not in ('GET', 'HEAD')):
method not in ('GET', 'HEAD')):
method = 'GET'
prepared_request.method = method
@@ -286,8 +289,8 @@ class Session(SessionRedirectMixin):
for (k, v) in env_proxies.items():
proxies.setdefault(k, v)
# Set environment's basic authentication.
if not auth:
# Set environment's basic authentication if not explicitly set.
if not auth and not self.auth:
auth = get_netrc_auth(url)
# Look for configuration.
@@ -467,7 +470,7 @@ class Session(SessionRedirectMixin):
"""Returns the appropriate connnection adapter for the given URL."""
for (prefix, adapter) in self.adapters.items():
if url.startswith(prefix):
if url.lower().startswith(prefix):
return adapter
# Nothing matches :-/
+2 -1
View File
@@ -18,7 +18,8 @@ _codes = {
205: ('reset_content', 'reset'),
206: ('partial_content', 'partial'),
207: ('multi_status', 'multiple_status', 'multi_stati', 'multiple_stati'),
208: ('im_used',),
208: ('already_reported',),
226: ('im_used',),
# Redirection.
300: ('multiple_choices',),
+14 -16
View File
@@ -22,6 +22,7 @@ from . import __version__
from . import certs
from .compat import parse_http_list as _parse_list_header
from .compat import quote, urlparse, bytes, str, OrderedDict, urlunparse
from .compat import getproxies, proxy_bypass
from .cookies import RequestsCookieJar, cookiejar_from_dict
from .structures import CaseInsensitiveDict
@@ -301,7 +302,7 @@ def stream_decode_response_unicode(iterator, r):
rv = decoder.decode(chunk)
if rv:
yield rv
rv = decoder.decode('', final=True)
rv = decoder.decode(b'', final=True)
if rv:
yield rv
@@ -386,37 +387,34 @@ def requote_uri(uri):
def get_environ_proxies(url):
"""Return a dict of environment proxies."""
proxy_keys = [
'all',
'http',
'https',
'ftp',
'socks'
]
get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper())
# First check whether no_proxy is defined. If it is, check that the URL
# we're getting isn't in the no_proxy list.
no_proxy = get_proxy('no_proxy')
netloc = urlparse(url).netloc
if no_proxy:
# We need to check whether we match here. We need to see if we match
# the end of the netloc, both with and without the port.
no_proxy = no_proxy.split(',')
netloc = urlparse(url).netloc
for host in no_proxy:
if netloc.endswith(host) or netloc.split(':')[0].endswith(host):
# The URL does match something in no_proxy, so we don't want
# to apply the proxies on this URL.
return {}
# If the system proxy settings indicate that this URL should be bypassed,
# don't proxy.
if proxy_bypass(netloc):
return {}
# If we get here, we either didn't have no_proxy set or we're not going
# anywhere that no_proxy applies to.
proxies = [(key, get_proxy(key + '_proxy')) for key in proxy_keys]
return dict([(key, val) for (key, val) in proxies if val])
# anywhere that no_proxy applies to, and the system settings don't require
# bypassing the proxy for the current URL.
return getproxies()
def default_user_agent():
"""Return a string representing the default user agent."""
+76 -1
View File
@@ -87,6 +87,34 @@ class RequestsTestCase(unittest.TestCase):
self.assertEqual(request.url,
"http://example.com/path?key=value&a=b#fragment")
def test_mixed_case_scheme_acceptable(self):
s = requests.Session()
r = requests.Request('GET', 'http://httpbin.org/get')
r = s.send(r.prepare())
self.assertEqual(r.status_code,200)
s = requests.Session()
r = requests.Request('GET', 'HTTP://httpbin.org/get')
r = s.send(r.prepare())
self.assertEqual(r.status_code,200)
r = requests.Request('GET', 'hTTp://httpbin.org/get')
r = s.send(r.prepare())
self.assertEqual(r.status_code,200)
r = requests.Request('GET', 'HttP://httpbin.org/get')
r = s.send(r.prepare())
self.assertEqual(r.status_code,200)
r = requests.Request('GET', 'https://httpbin.org/get')
r = s.send(r.prepare())
self.assertEqual(r.status_code,200)
r = requests.Request('GET', 'HTTPS://httpbin.org/get')
r = s.send(r.prepare())
self.assertEqual(r.status_code,200)
r = requests.Request('GET', 'hTTps://httpbin.org/get')
r = s.send(r.prepare())
self.assertEqual(r.status_code,200)
r = requests.Request('GET', 'HttPs://httpbin.org/get')
r = s.send(r.prepare())
self.assertEqual(r.status_code,200)
def test_HTTP_200_OK_GET_ALTERNATIVE(self):
r = requests.Request('GET', httpbin('get'))
s = requests.Session()
@@ -142,6 +170,11 @@ class RequestsTestCase(unittest.TestCase):
)
assert 'foo' not in s.cookies
def test_cookie_quote_wrapped(self):
s = requests.session()
s.get(httpbin('cookies/set?foo="bar:baz"'))
self.assertTrue(s.cookies['foo'] == '"bar:baz"')
def test_request_cookie_overrides_session_cookie(self):
s = requests.session()
s.cookies['foo'] = 'bar'
@@ -160,7 +193,13 @@ class RequestsTestCase(unittest.TestCase):
assert r.json()['cookies']['foo'] == 'bar'
# Make sure the session cj is still the custom one
assert s.cookies is cj
def test_requests_in_history_are_not_overridden(self):
resp = requests.get(httpbin('redirect/3'))
urls = [r.url for r in resp.history]
req_urls = [r.request.url for r in resp.history]
self.assertEquals(urls, req_urls)
def test_user_agent_transfers(self):
heads = {
@@ -200,6 +239,34 @@ class RequestsTestCase(unittest.TestCase):
r = s.get(url)
self.assertEqual(r.status_code, 200)
def test_basicauth_with_netrc(self):
auth = ('user', 'pass')
wrong_auth = ('wronguser', 'wrongpass')
url = httpbin('basic-auth', 'user', 'pass')
def get_netrc_auth_mock(url):
return auth
requests.sessions.get_netrc_auth = get_netrc_auth_mock
# Should use netrc and work.
r = requests.get(url)
self.assertEqual(r.status_code, 200)
# Given auth should override and fail.
r = requests.get(url, auth=wrong_auth)
self.assertEqual(r.status_code, 401)
s = requests.session()
# Should use netrc and work.
r = s.get(url)
self.assertEqual(r.status_code, 200)
# Given auth should override and fail.
s.auth = wrong_auth
r = s.get(url)
self.assertEqual(r.status_code, 401)
def test_DIGEST_HTTP_200_OK_GET(self):
auth = HTTPDigestAuth('user', 'pass')
@@ -496,6 +563,14 @@ class RequestsTestCase(unittest.TestCase):
'application/json'
)
def test_uppercase_scheme(self):
r = requests.get('HTTP://example.com/')
self.assertEqual(r.status_code, 200)
def test_uppercase_scheme_redirect(self):
r = requests.get(httpbin('redirect-to'), params={'url': 'HTTP://example.com/'})
self.assertEqual(r.status_code, 200)
def test_transport_adapter_ordering(self):
s = requests.Session()
order = ['https://', 'http://']