kennethreitz/requests
1
0
Fork 0
mirror of https://github.com/kennethreitz/requests.git synced 2026-06-05 22:50:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

Browse Source 
Conflicts:
	requests/sessions.py
This commit is contained in:
sprt
2013-01-23 14:42:45 +01:00
parent a721d590b4 0c09d16b29
commit c4ad8afba9
16 changed files with 131 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+1
View File
@@ -3,6 +3,7 @@ python:
- 2.6
- 2.7
- 3.2
- 3.3
env: HTTPBIN_URL=http://httpbin.org/
script: make test
install:
docs/user/advanced.rst
+2 -2
View File
@@ -144,7 +144,7 @@ Streaming Uploads
Requests supports streaming uploads, which allow you to send large streams or files without reading them into memory. To stream and upload, simply provide a file-like object for your body::
with open('massive-body') as f:
request.post('http://some.url/streamed', data=f)
requests.post('http://some.url/streamed', data=f)
Chunk-Encoded Requests
@@ -157,7 +157,7 @@ Requests also supports Chunked transfer encoding for outgoing and incoming reque
yield 'hi'
yield 'there'
request.post('http://some.url/chunked', data=gen())
requests.post('http://some.url/chunked', data=gen())
Event Hooks
docs/user/quickstart.rst
+3 -3
View File
@@ -320,11 +320,11 @@ Cookies
If a response contains some Cookies, you can get quick access to them::
>>> url = 'http://httpbin.org/cookies/set/requests-is/awesome'
>>> url = 'http://example.com/some/cookie/setting/url'
>>> r = requests.get(url)
>>> r.cookies['requests-is']
'awesome'
>>> r.cookies['example_cookie_name']
'example_cookie_value'
To send your own cookies to the server, you can use the ``cookies``
parameter::
requests/adapters.py
+2 -5
View File
@@ -11,9 +11,8 @@ and maintain connections.
import socket
from .models import Response
from .packages.urllib3.poolmanager import PoolManager, proxy_from_url
from .packages.urllib3.poolmanager import PoolManager, ProxyManager
from .packages.urllib3.response import HTTPResponse
from .hooks import dispatch_hook
from .compat import urlparse, basestring, urldefrag
from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers,
prepend_scheme_if_needed)
@@ -109,8 +108,6 @@ class HTTPAdapter(BaseAdapter):
response.request = req
response.connection = self
# Run the Response hook.
response = dispatch_hook('response', req.hooks, response)
return response
def get_connection(self, url, proxies=None):
@@ -120,7 +117,7 @@ class HTTPAdapter(BaseAdapter):
if proxy:
proxy = prepend_scheme_if_needed(proxy, urlparse(url).scheme)
conn = proxy_from_url(proxy)
conn = ProxyManager(self.poolmanager.connection_from_url(proxy))
else:
conn = self.poolmanager.connection_from_url(url)
requests/auth.py
+10 -6
View File
@@ -68,18 +68,21 @@ class HTTPDigestAuth(AuthBase):
realm = self.chal['realm']
nonce = self.chal['nonce']
qop = self.chal.get('qop')
algorithm = self.chal.get('algorithm', 'MD5')
opaque = self.chal.get('opaque', None)
algorithm = self.chal.get('algorithm')
opaque = self.chal.get('opaque')
algorithm = algorithm.upper()
if algorithm is None:
_algorithm = 'MD5'
else:
_algorithm = algorithm.upper()
# lambdas assume digest modules are imported at the top level
if algorithm == 'MD5':
if _algorithm == 'MD5':
def md5_utf8(x):
if isinstance(x, str):
x = x.encode('utf-8')
return hashlib.md5(x).hexdigest()
hash_utf8 = md5_utf8
elif algorithm == 'SHA':
elif _algorithm == 'SHA':
def sha_utf8(x):
if isinstance(x, str):
x = x.encode('utf-8')
@@ -129,9 +132,10 @@ class HTTPDigestAuth(AuthBase):
'response="%s"' % (self.username, realm, nonce, path, respdig)
if opaque:
base += ', opaque="%s"' % opaque
if algorithm:
base += ', algorithm="%s"' % algorithm
if entdig:
base += ', digest="%s"' % entdig
base += ', algorithm="%s"' % algorithm
if qop:
base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce)
requests/cacert.pem
-48
View File
@@ -1603,54 +1603,6 @@ vFcj4jjSm2jzVhKIT0J8uDHEtdvkyCE06UgRNe76x5JXxZ805Mf29w4LTJxoeHtxMcfrHuBnQfO3
oKfN5XozNmr6mis=
-----END CERTIFICATE-----
TURKTRUST Certificate Services Provider Root 1
==============================================
-----BEGIN CERTIFICATE-----
MIID+zCCAuOgAwIBAgIBATANBgkqhkiG9w0BAQUFADCBtzE/MD0GA1UEAww2VMOcUktUUlVTVCBF
bGVrdHJvbmlrIFNlcnRpZmlrYSBIaXptZXQgU2HEn2xhecSxY8Sxc8SxMQswCQYDVQQGDAJUUjEP
MA0GA1UEBwwGQU5LQVJBMVYwVAYDVQQKDE0oYykgMjAwNSBUw5xSS1RSVVNUIEJpbGdpIMSwbGV0
acWfaW0gdmUgQmlsacWfaW0gR8O8dmVubGnEn2kgSGl6bWV0bGVyaSBBLsWeLjAeFw0wNTA1MTMx
MDI3MTdaFw0xNTAzMjIxMDI3MTdaMIG3MT8wPQYDVQQDDDZUw5xSS1RSVVNUIEVsZWt0cm9uaWsg
U2VydGlmaWthIEhpem1ldCBTYcSfbGF5xLFjxLFzxLExCzAJBgNVBAYMAlRSMQ8wDQYDVQQHDAZB
TktBUkExVjBUBgNVBAoMTShjKSAyMDA1IFTDnFJLVFJVU1QgQmlsZ2kgxLBsZXRpxZ9pbSB2ZSBC
aWxpxZ9pbSBHw7x2ZW5sacSfaSBIaXptZXRsZXJpIEEuxZ4uMIIBIjANBgkqhkiG9w0BAQEFAAOC
AQ8AMIIBCgKCAQEAylIF1mMD2Bxf3dJ7XfIMYGFbazt0K3gNfUW9InTojAPBxhEqPZW8qZSwu5GX
yGl8hMW0kWxsE2qkVa2kheiVfrMArwDCBRj1cJ02i67L5BuBf5OI+2pVu32Fks66WJ/bMsW9Xe8i
Si9BB35JYbOG7E6mQW6EvAPs9TscyB/C7qju6hJKjRTP8wrgUDn5CDX4EVmt5yLqS8oUBt5CurKZ
8y1UiBAG6uEaPj1nH/vO+3yC6BFdSsG5FOpU2WabfIl9BJpiyelSPJ6c79L1JuTm5Rh8i27fbMx4
W09ysstcP4wFjdFMjK2Sx+F4f2VsSQZQLJ4ywtdKxnWKWU51b0dewQIDAQABoxAwDjAMBgNVHRME
BTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQAV9VX/N5aAWSGk/KEVTCD21F/aAyT8z5Aa9CEKmu46
sWrv7/hg0Uw2ZkUd82YCdAR7kjCo3gp2D++Vbr3JN+YaDayJSFvMgzbC9UZcWYJWtNX+I7TYVBxE
q8Sn5RTOPEFhfEPmzcSBCYsk+1Ql1haolgxnB2+zUEfjHCQo3SqYpGH+2+oSN7wBGjSFvW5P55Fy
B0SFHljKVETd96y5y4khctuPwGkplyqjrhgjlxxBKot8KsF8kOipKMDTkcatKIdAaLX/7KfS0zgY
nNN9aV3wxqUeJBujR/xpB2jn5Jq07Q+hh4cCzofSSE7hvP/L8XKSRGQDJereW26fyfJOrN3H
-----END CERTIFICATE-----
TURKTRUST Certificate Services Provider Root 2
==============================================
-----BEGIN CERTIFICATE-----
MIIEPDCCAySgAwIBAgIBATANBgkqhkiG9w0BAQUFADCBvjE/MD0GA1UEAww2VMOcUktUUlVTVCBF
bGVrdHJvbmlrIFNlcnRpZmlrYSBIaXptZXQgU2HEn2xhecSxY8Sxc8SxMQswCQYDVQQGEwJUUjEP
MA0GA1UEBwwGQW5rYXJhMV0wWwYDVQQKDFRUw5xSS1RSVVNUIEJpbGdpIMSwbGV0acWfaW0gdmUg
QmlsacWfaW0gR8O8dmVubGnEn2kgSGl6bWV0bGVyaSBBLsWeLiAoYykgS2FzxLFtIDIwMDUwHhcN
MDUxMTA3MTAwNzU3WhcNMTUwOTE2MTAwNzU3WjCBvjE/MD0GA1UEAww2VMOcUktUUlVTVCBFbGVr
dHJvbmlrIFNlcnRpZmlrYSBIaXptZXQgU2HEn2xhecSxY8Sxc8SxMQswCQYDVQQGEwJUUjEPMA0G
A1UEBwwGQW5rYXJhMV0wWwYDVQQKDFRUw5xSS1RSVVNUIEJpbGdpIMSwbGV0acWfaW0gdmUgQmls
acWfaW0gR8O8dmVubGnEn2kgSGl6bWV0bGVyaSBBLsWeLiAoYykgS2FzxLFtIDIwMDUwggEiMA0G
CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCpNn7DkUNMwxmYCMjHWHtPFoylzkkBH3MOrHUTpvqe
LCDe2JAOCtFp0if7qnefJ1Il4std2NiDUBd9irWCPwSOtNXwSadktx4uXyCcUHVPr+G1QRT0mJKI
x+XlZEdhR3n9wFHxwZnn3M5q+6+1ATDcRhzviuyV79z/rxAc653YsKpqhRgNF8k+v/Gb0AmJQv2g
QrSdiVFVKc8bcLyEVK3BEx+Y9C52YItdP5qtygy/p1Zbj3e41Z55SZI/4PGXJHpsmxcPbe9TmJEr
5A++WXkHeLuXlfSfadRYhwqp48y2WBmfJiGxxFmNskF1wK1pzpwACPI2/z7woQ8arBT9pmAPAgMB
AAGjQzBBMB0GA1UdDgQWBBTZN7NOBf3Zz58SFq62iS/rJTqIHDAPBgNVHQ8BAf8EBQMDBwYAMA8G
A1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAHJglrfJ3NgpXiOFX7KzLXb7iNcX/ntt
Rbj2hWyfIvwqECLsqrkw9qtY1jkQMZkpAL2JZkH7dN6RwRgLn7Vhy506vvWolKMiVW4XSf/SKfE4
Jl3vpao6+XF75tpYHdN0wgH6PmlYX63LaL4ULptswLbcoCb6dxriJNoaN+BnrdFzgw2lGh1uEpJ+
hGIAF728JRhX8tepb1mIvDS3LoV4nZbcFMMsilKbloxSZj2GFotHuFEJjOp9zYhys2AzsfAKRO8P
9Qk3iCQOLGsgOqL6EfJANZxEaGM7rDNvY7wsu/LSy3Z9fYjYHcgFHW68lKlmjHdxx/qR+i9Rnuk5
UrbnBEI=
-----END CERTIFICATE-----
SwissSign Gold CA - G2
======================
-----BEGIN CERTIFICATE-----
requests/certs.py
+1 -9
View File
@@ -14,17 +14,9 @@ packaged CA bundle.
import os.path
certifi = None
try:
import certifi
except ImportError:
pass
def where():
"""Return the preferred certificate bundle."""
if certifi:
return certifi.where()
# vendored bundle inside Requests
return os.path.join(os.path.dirname(__file__), 'cacert.pem')
if __name__ == '__main__':
requests/models.py
+17 -11
View File
@@ -29,7 +29,7 @@ from .compat import (
REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
CONTENT_CHUNK_SIZE = 10 * 1024
ITER_CHUNK_SIZE = 10 * 1024
ITER_CHUNK_SIZE = 512
log = logging.getLogger(__name__)
@@ -121,7 +121,7 @@ class RequestEncodingMixin(object):
fp = StringIO(fp)
if isinstance(fp, bytes):
fp = BytesIO(fp)
if ft:
new_v = (fn, fp.read(), ft)
else:
@@ -284,7 +284,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
# Support for unicode domain names and paths.
scheme, netloc, path, _params, query, fragment = urlparse(url)
if not scheme:
if not (scheme and netloc):
raise MissingSchema("Invalid URL %r: No schema supplied" % url)
try:
@@ -347,7 +347,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
])
try:
length = super_len(data)
length = str(super_len(data))
except (TypeError, AttributeError):
length = False
@@ -376,13 +376,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
else:
content_type = 'application/x-www-form-urlencoded'
self.headers['Content-Length'] = '0'
if hasattr(body, 'seek') and hasattr(body, 'tell'):
body.seek(0, 2)
self.headers['Content-Length'] = str(body.tell())
body.seek(0, 0)
elif body is not None:
self.headers['Content-Length'] = str(len(body))
self.prepare_content_length(body)
# Add content-type if it wasn't explicitly provided.
if (content_type) and (not 'content-type' in self.headers):
@@ -390,6 +384,15 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
self.body = body
def prepare_content_length(self, body):
self.headers['Content-Length'] = '0'
if hasattr(body, 'seek') and hasattr(body, 'tell'):
body.seek(0, 2)
self.headers['Content-Length'] = str(body.tell())
body.seek(0, 0)
elif body is not None:
self.headers['Content-Length'] = str(len(body))
def prepare_auth(self, auth):
"""Prepares the given HTTP auth data."""
if auth:
@@ -403,6 +406,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
# Update self to reflect the auth changes.
self.__dict__.update(r.__dict__)
# Recompute Content-Length
self.prepare_content_length(self.body)
def prepare_cookies(self, cookies):
"""Prepares the given HTTP cookie data."""
requests/packages/urllib3/connectionpool.py
+11 -15
View File
@@ -9,6 +9,7 @@ import socket
import errno
from socket import error as SocketError, timeout as SocketTimeout
from .util import resolve_cert_reqs, resolve_ssl_version
try: # Python 3
from http.client import HTTPConnection, HTTPException
@@ -80,31 +81,29 @@ class VerifiedHTTPSConnection(HTTPSConnection):
ssl_version = None
def set_cert(self, key_file=None, cert_file=None,
cert_reqs='CERT_NONE', ca_certs=None):
ssl_req_scheme = {
'CERT_NONE': ssl.CERT_NONE,
'CERT_OPTIONAL': ssl.CERT_OPTIONAL,
'CERT_REQUIRED': ssl.CERT_REQUIRED
}
cert_reqs=None, ca_certs=None):
self.key_file = key_file
self.cert_file = cert_file
self.cert_reqs = ssl_req_scheme.get(cert_reqs) or ssl.CERT_NONE
self.cert_reqs = cert_reqs
self.ca_certs = ca_certs
def connect(self):
# Add certificate verification
sock = socket.create_connection((self.host, self.port), self.timeout)
resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs)
resolved_ssl_version = resolve_ssl_version(self.ssl_version)
# Wrap socket using verification with the root certs in
# trusted_root_certs
self.sock = ssl_wrap_socket(sock, self.key_file, self.cert_file,
cert_reqs=self.cert_reqs,
cert_reqs=resolved_cert_reqs,
ca_certs=self.ca_certs,
server_hostname=self.host,
ssl_version=self.ssl_version)
ssl_version=resolved_ssl_version)
if self.ca_certs:
if resolved_cert_reqs != ssl.CERT_NONE:
match_hostname(self.sock.getpeercert(), self.host)
@@ -514,7 +513,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
strict=False, timeout=None, maxsize=1,
block=False, headers=None,
key_file=None, cert_file=None,
cert_reqs='CERT_NONE', ca_certs=None, ssl_version=None):
cert_reqs=None, ca_certs=None, ssl_version=None):
HTTPConnectionPool.__init__(self, host, port,
strict, timeout, maxsize,
@@ -548,10 +547,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
connection.set_cert(key_file=self.key_file, cert_file=self.cert_file,
cert_reqs=self.cert_reqs, ca_certs=self.ca_certs)
if self.ssl_version is None:
connection.ssl_version = ssl.PROTOCOL_SSLv23
else:
connection.ssl_version = self.ssl_version
connection.ssl_version = self.ssl_version
return connection
requests/packages/urllib3/poolmanager.py
+13 -4
View File
@@ -58,6 +58,17 @@ class PoolManager(RequestMethods):
self.pools = RecentlyUsedContainer(num_pools,
dispose_func=lambda p: p.close())
def _new_pool(self, scheme, host, port):
"""
Create a new :class:`ConnectionPool` based on host, port and scheme.
This method is used to actually create the connection pools handed out
by :meth:`connection_from_url` and companion methods. It is intended
to be overridden for customization.
"""
pool_cls = pool_classes_by_scheme[scheme]
return pool_cls(host, port, **self.connection_pool_kw)
def clear(self):
"""
Empty our store of pools and direct them all to close.
@@ -74,6 +85,7 @@ class PoolManager(RequestMethods):
If ``port`` isn't given, it will be derived from the ``scheme`` using
``urllib3.connectionpool.port_by_scheme``.
"""
scheme = scheme or 'http'
port = port or port_by_scheme.get(scheme, 80)
pool_key = (scheme, host, port)
@@ -85,11 +97,8 @@ class PoolManager(RequestMethods):
return pool
# Make a fresh ConnectionPool of the desired type
pool_cls = pool_classes_by_scheme[scheme]
pool = pool_cls(host, port, **self.connection_pool_kw)
pool = self._new_pool(scheme, host, port)
self.pools[pool_key] = pool
return pool
def connection_from_url(self, url):
requests/packages/urllib3/response.py
+11 -1
View File
@@ -145,7 +145,17 @@ class HTTPResponse(object):
# cStringIO doesn't like amt=None
data = self._fp.read()
else:
return self._fp.read(amt)
data = self._fp.read(amt)
if amt != 0 and not data: # Platform-specific: Buggy versions of Python.
# Close the connection when no data is returned
#
# This is redundant to what httplib/http.client _should_
# already do. However, versions of python released before
# December 15, 2012 (http://bugs.python.org/issue16298) do not
# properly close the connection in all cases. There is no harm
# in redundantly calling close.
self._fp.close()
return data
try:
if decode_content and decoder:
requests/packages/urllib3/util.py
+46 -6
View File
@@ -22,6 +22,7 @@ try: # Test for SSL features
SSLContext = None
HAS_SNI = False
import ssl
from ssl import wrap_socket, CERT_NONE, SSLError, PROTOCOL_SSLv23
from ssl import SSLContext # Modern SSL?
from ssl import HAS_SNI # Has SNI?
@@ -263,10 +264,48 @@ def is_connection_dropped(conn):
return True
def resolve_cert_reqs(candidate):
"""
Resolves the argument to a numeric constant, which can be passed to
the wrap_socket function/method from the ssl module.
Defaults to :data:`ssl.CERT_NONE`.
If given a string it is assumed to be the name of the constant in the
:mod:`ssl` module or its abbrevation.
(So you can specify `REQUIRED` instead of `CERT_REQUIRED`.
If it's neither `None` nor a string we assume it is already the numeric
constant which can directly be passed to wrap_socket.
"""
if candidate is None:
return CERT_NONE
if isinstance(candidate, str):
res = getattr(ssl, candidate, None)
if res is None:
res = getattr(ssl, 'CERT_' + candidate)
return res
return candidate
def resolve_ssl_version(candidate):
"""
like resolve_cert_reqs
"""
if candidate is None:
return PROTOCOL_SSLv23
if isinstance(candidate, str):
res = getattr(ssl, candidate, None)
if res is None:
res = getattr(ssl, 'PROTOCOL_' + candidate)
return res
return candidate
if SSLContext is not None: # Python 3.2+
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=CERT_NONE,
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
ca_certs=None, server_hostname=None,
ssl_version=PROTOCOL_SSLv23):
ssl_version=None):
"""
All arguments except `server_hostname` have the same meaning as for
:func:`ssl.wrap_socket`
@@ -279,8 +318,9 @@ if SSLContext is not None: # Python 3.2+
if ca_certs:
try:
context.load_verify_locations(ca_certs)
except TypeError as e: # Reraise as SSLError
# FIXME: This block needs a test.
# Py32 raises IOError
# Py33 raises FileNotFoundError
except Exception as e: # Reraise as SSLError
raise SSLError(e)
if certfile:
# FIXME: This block needs a test.
@@ -290,9 +330,9 @@ if SSLContext is not None: # Python 3.2+
return context.wrap_socket(sock)
else: # Python 3.1 and earlier
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=CERT_NONE,
def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None,
ca_certs=None, server_hostname=None,
ssl_version=PROTOCOL_SSLv23):
ssl_version=None):
return wrap_socket(sock, keyfile=keyfile, certfile=certfile,
ca_certs=ca_certs, cert_reqs=cert_reqs,
ssl_version=ssl_version)
requests/sessions.py
+7
View File
@@ -275,6 +275,10 @@ class Session(SessionRedirectMixin):
# Prepare the Request.
prep = req.prepare()
# If auth hooks are present, they aren't passed to `dispatch_hook`
# As such, we need to update the original hooks dictionary with them
hooks.update(prep.hooks)
# Send the request.
resp = self.send(prep, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)
@@ -294,6 +298,9 @@ class Session(SessionRedirectMixin):
resp = history.pop()
resp.history = tuple(history)
# Response manipulation hook.
resp = dispatch_hook('response', hooks, resp)
return resp
def get(self, url, **kwargs):
requests/utils.py
-2
View File
@@ -28,8 +28,6 @@ _hush_pyflakes = (RequestsCookieJar,)
NETRC_FILES = ('.netrc', '_netrc')
# if certifi is installed, use its CA bundle;
# otherwise, try and use the OS bundle
DEFAULT_CA_BUNDLE_PATH = certs.where()
def dict_to_sequence(d):
setup.py
+1 -1
View File
@@ -17,7 +17,7 @@ if sys.argv[-1] == 'publish':
packages = [
'requests',
'requests.packages',
'requests.packages.charade',
'requests.packages.charade',
'requests.packages.urllib3',
'requests.packages.urllib3.packages',
'requests.packages.urllib3.packages.ssl_match_hostname'
test_requests.py
+6 -2
View File
@@ -9,6 +9,12 @@ import unittest
import requests
from requests.auth import HTTPDigestAuth
from requests.compat import str
try:
import StringIO
except ImportError:
import io as StringIO
HTTPBIN = os.environ.get('HTTPBIN_URL', 'http://httpbin.org/')
@@ -131,8 +137,6 @@ class RequestsTestCase(unittest.TestCase):
self.assertEqual(r.status_code, 200)
def test_BASICAUTH_TUPLE_HTTP_200_OK_GET(self):
auth = ('user', 'pass')
url = httpbin('basic-auth', 'user', 'pass')