From a950322b0794a451491d51bd8596623964c5b323 Mon Sep 17 00:00:00 2001 From: Jud Date: Fri, 11 Jan 2013 11:42:02 -0500 Subject: [PATCH 01/16] Fix spacing in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 795774e9..52dfe025 100755 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ if sys.argv[-1] == 'publish': packages = [ 'requests', 'requests.packages', - 'requests.packages.charade', + 'requests.packages.charade', 'requests.packages.urllib3', 'requests.packages.urllib3.packages', 'requests.packages.urllib3.packages.ssl_match_hostname' From 628e393b9a251c3d2b9910c697b9e4ac4f3d8d6a Mon Sep 17 00:00:00 2001 From: Matt McClure Date: Fri, 11 Jan 2013 15:04:47 -0500 Subject: [PATCH 02/16] Resolves the parts of #1096 in requests proper. --- requests/models.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/requests/models.py b/requests/models.py index 5202e6f4..b7d52cd7 100644 --- a/requests/models.py +++ b/requests/models.py @@ -375,13 +375,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): else: content_type = 'application/x-www-form-urlencoded' - self.headers['Content-Length'] = '0' - if hasattr(body, 'seek') and hasattr(body, 'tell'): - body.seek(0, 2) - self.headers['Content-Length'] = str(body.tell()) - body.seek(0, 0) - elif body is not None: - self.headers['Content-Length'] = str(len(body)) + self.prepare_content_length(body) # Add content-type if it wasn't explicitly provided. if (content_type) and (not 'content-type' in self.headers): @@ -389,6 +383,15 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): self.body = body + def prepare_content_length(self, body): + self.headers['Content-Length'] = '0' + if hasattr(body, 'seek') and hasattr(body, 'tell'): + body.seek(0, 2) + self.headers['Content-Length'] = str(body.tell()) + body.seek(0, 0) + elif body is not None: + self.headers['Content-Length'] = str(len(body)) + def prepare_auth(self, auth): """Prepares the given HTTP auth data.""" if auth: @@ -402,6 +405,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): # Update self to reflect the auth changes. self.__dict__.update(r.__dict__) + # Recompute Content-Length + self.prepare_content_length(self.body) + def prepare_cookies(self, cookies): """Prepares the given HTTP cookie data.""" From b66427eab95298a4a42d0801873b3d3e5d6c3b0f Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Thu, 17 Jan 2013 02:02:49 -0800 Subject: [PATCH 03/16] remove the TURKTRUST root certificates, as per #1102 --- requests/cacert.pem | 48 --------------------------------------------- 1 file changed, 48 deletions(-) diff --git a/requests/cacert.pem b/requests/cacert.pem index 7da84474..504fdccf 100644 --- a/requests/cacert.pem +++ b/requests/cacert.pem @@ -1603,54 +1603,6 @@ vFcj4jjSm2jzVhKIT0J8uDHEtdvkyCE06UgRNe76x5JXxZ805Mf29w4LTJxoeHtxMcfrHuBnQfO3 oKfN5XozNmr6mis= -----END CERTIFICATE----- -TURKTRUST Certificate Services Provider Root 1 -============================================== ------BEGIN CERTIFICATE----- -MIID+zCCAuOgAwIBAgIBATANBgkqhkiG9w0BAQUFADCBtzE/MD0GA1UEAww2VMOcUktUUlVTVCBF -bGVrdHJvbmlrIFNlcnRpZmlrYSBIaXptZXQgU2HEn2xhecSxY8Sxc8SxMQswCQYDVQQGDAJUUjEP -MA0GA1UEBwwGQU5LQVJBMVYwVAYDVQQKDE0oYykgMjAwNSBUw5xSS1RSVVNUIEJpbGdpIMSwbGV0 -acWfaW0gdmUgQmlsacWfaW0gR8O8dmVubGnEn2kgSGl6bWV0bGVyaSBBLsWeLjAeFw0wNTA1MTMx -MDI3MTdaFw0xNTAzMjIxMDI3MTdaMIG3MT8wPQYDVQQDDDZUw5xSS1RSVVNUIEVsZWt0cm9uaWsg -U2VydGlmaWthIEhpem1ldCBTYcSfbGF5xLFjxLFzxLExCzAJBgNVBAYMAlRSMQ8wDQYDVQQHDAZB -TktBUkExVjBUBgNVBAoMTShjKSAyMDA1IFTDnFJLVFJVU1QgQmlsZ2kgxLBsZXRpxZ9pbSB2ZSBC -aWxpxZ9pbSBHw7x2ZW5sacSfaSBIaXptZXRsZXJpIEEuxZ4uMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEAylIF1mMD2Bxf3dJ7XfIMYGFbazt0K3gNfUW9InTojAPBxhEqPZW8qZSwu5GX -yGl8hMW0kWxsE2qkVa2kheiVfrMArwDCBRj1cJ02i67L5BuBf5OI+2pVu32Fks66WJ/bMsW9Xe8i -Si9BB35JYbOG7E6mQW6EvAPs9TscyB/C7qju6hJKjRTP8wrgUDn5CDX4EVmt5yLqS8oUBt5CurKZ -8y1UiBAG6uEaPj1nH/vO+3yC6BFdSsG5FOpU2WabfIl9BJpiyelSPJ6c79L1JuTm5Rh8i27fbMx4 -W09ysstcP4wFjdFMjK2Sx+F4f2VsSQZQLJ4ywtdKxnWKWU51b0dewQIDAQABoxAwDjAMBgNVHRME -BTADAQH/MA0GCSqGSIb3DQEBBQUAA4IBAQAV9VX/N5aAWSGk/KEVTCD21F/aAyT8z5Aa9CEKmu46 -sWrv7/hg0Uw2ZkUd82YCdAR7kjCo3gp2D++Vbr3JN+YaDayJSFvMgzbC9UZcWYJWtNX+I7TYVBxE -q8Sn5RTOPEFhfEPmzcSBCYsk+1Ql1haolgxnB2+zUEfjHCQo3SqYpGH+2+oSN7wBGjSFvW5P55Fy -B0SFHljKVETd96y5y4khctuPwGkplyqjrhgjlxxBKot8KsF8kOipKMDTkcatKIdAaLX/7KfS0zgY -nNN9aV3wxqUeJBujR/xpB2jn5Jq07Q+hh4cCzofSSE7hvP/L8XKSRGQDJereW26fyfJOrN3H ------END CERTIFICATE----- - -TURKTRUST Certificate Services Provider Root 2 -============================================== ------BEGIN CERTIFICATE----- -MIIEPDCCAySgAwIBAgIBATANBgkqhkiG9w0BAQUFADCBvjE/MD0GA1UEAww2VMOcUktUUlVTVCBF -bGVrdHJvbmlrIFNlcnRpZmlrYSBIaXptZXQgU2HEn2xhecSxY8Sxc8SxMQswCQYDVQQGEwJUUjEP -MA0GA1UEBwwGQW5rYXJhMV0wWwYDVQQKDFRUw5xSS1RSVVNUIEJpbGdpIMSwbGV0acWfaW0gdmUg -QmlsacWfaW0gR8O8dmVubGnEn2kgSGl6bWV0bGVyaSBBLsWeLiAoYykgS2FzxLFtIDIwMDUwHhcN -MDUxMTA3MTAwNzU3WhcNMTUwOTE2MTAwNzU3WjCBvjE/MD0GA1UEAww2VMOcUktUUlVTVCBFbGVr -dHJvbmlrIFNlcnRpZmlrYSBIaXptZXQgU2HEn2xhecSxY8Sxc8SxMQswCQYDVQQGEwJUUjEPMA0G -A1UEBwwGQW5rYXJhMV0wWwYDVQQKDFRUw5xSS1RSVVNUIEJpbGdpIMSwbGV0acWfaW0gdmUgQmls -acWfaW0gR8O8dmVubGnEn2kgSGl6bWV0bGVyaSBBLsWeLiAoYykgS2FzxLFtIDIwMDUwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCpNn7DkUNMwxmYCMjHWHtPFoylzkkBH3MOrHUTpvqe -LCDe2JAOCtFp0if7qnefJ1Il4std2NiDUBd9irWCPwSOtNXwSadktx4uXyCcUHVPr+G1QRT0mJKI -x+XlZEdhR3n9wFHxwZnn3M5q+6+1ATDcRhzviuyV79z/rxAc653YsKpqhRgNF8k+v/Gb0AmJQv2g -QrSdiVFVKc8bcLyEVK3BEx+Y9C52YItdP5qtygy/p1Zbj3e41Z55SZI/4PGXJHpsmxcPbe9TmJEr -5A++WXkHeLuXlfSfadRYhwqp48y2WBmfJiGxxFmNskF1wK1pzpwACPI2/z7woQ8arBT9pmAPAgMB -AAGjQzBBMB0GA1UdDgQWBBTZN7NOBf3Zz58SFq62iS/rJTqIHDAPBgNVHQ8BAf8EBQMDBwYAMA8G -A1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAHJglrfJ3NgpXiOFX7KzLXb7iNcX/ntt -Rbj2hWyfIvwqECLsqrkw9qtY1jkQMZkpAL2JZkH7dN6RwRgLn7Vhy506vvWolKMiVW4XSf/SKfE4 -Jl3vpao6+XF75tpYHdN0wgH6PmlYX63LaL4ULptswLbcoCb6dxriJNoaN+BnrdFzgw2lGh1uEpJ+ -hGIAF728JRhX8tepb1mIvDS3LoV4nZbcFMMsilKbloxSZj2GFotHuFEJjOp9zYhys2AzsfAKRO8P -9Qk3iCQOLGsgOqL6EfJANZxEaGM7rDNvY7wsu/LSy3Z9fYjYHcgFHW68lKlmjHdxx/qR+i9Rnuk5 -UrbnBEI= ------END CERTIFICATE----- - SwissSign Gold CA - G2 ====================== -----BEGIN CERTIFICATE----- From d3e6597f73c8e6f5fc88aa6afc62611fbd8be8bd Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Thu, 17 Jan 2013 19:28:28 +0000 Subject: [PATCH 04/16] Update docs with correct cookie behaviour. --- docs/user/quickstart.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 0e80eaa2..46ce5801 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -320,11 +320,11 @@ Cookies If a response contains some Cookies, you can get quick access to them:: - >>> url = 'http://httpbin.org/cookies/set/requests-is/awesome' + >>> url = 'http://example.com/some/cookie/setting/url' >>> r = requests.get(url) - >>> r.cookies['requests-is'] - 'awesome' + >>> r.cookies['example_cookie_name'] + 'example_cookie_value' To send your own cookies to the server, you can use the ``cookies`` parameter:: From 985a906c6c078356cddedd0b3331031a6a84960c Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Thu, 17 Jan 2013 20:40:28 -0800 Subject: [PATCH 05/16] Remove support for certifi As per #1105, certifi is being end-of-lifed. Requests will use either its own vendored bundle, or possibly (when packaged with OS distributions) an externally packaged bundle, which can be enabled by patching requests.certs.where(). --- requests/certs.py | 10 +--------- requests/utils.py | 2 -- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/requests/certs.py b/requests/certs.py index 81482767..752460c2 100644 --- a/requests/certs.py +++ b/requests/certs.py @@ -14,17 +14,9 @@ packaged CA bundle. import os.path -certifi = None -try: - import certifi -except ImportError: - pass - def where(): """Return the preferred certificate bundle.""" - if certifi: - return certifi.where() - + # vendored bundle inside Requests return os.path.join(os.path.dirname(__file__), 'cacert.pem') if __name__ == '__main__': diff --git a/requests/utils.py b/requests/utils.py index f5f6b959..34d92d25 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -28,8 +28,6 @@ _hush_pyflakes = (RequestsCookieJar,) NETRC_FILES = ('.netrc', '_netrc') -# if certifi is installed, use its CA bundle; -# otherwise, try and use the OS bundle DEFAULT_CA_BUNDLE_PATH = certs.where() def dict_to_sequence(d): From 347a52aa5ced6ff3fec8723ccfb8531fdbcf8d9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20Vask=C3=B3?= Date: Fri, 18 Jan 2013 13:41:14 +0100 Subject: [PATCH 06/16] Fixed proxy requests to pool connections. --- requests/adapters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 5f9d9c79..099a0011 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -11,7 +11,7 @@ and maintain connections. import socket from .models import Response -from .packages.urllib3.poolmanager import PoolManager, proxy_from_url +from .packages.urllib3.poolmanager import PoolManager, ProxyManager from .packages.urllib3.response import HTTPResponse from .hooks import dispatch_hook from .compat import urlparse, basestring, urldefrag @@ -120,7 +120,7 @@ class HTTPAdapter(BaseAdapter): if proxy: proxy = prepend_scheme_if_needed(proxy, urlparse(url).scheme) - conn = proxy_from_url(proxy) + conn = ProxyManager(self.poolmanager.connection_from_url(proxy)) else: conn = self.poolmanager.connection_from_url(url) From 89c8cbbe0c8e1db50746ca0f5e68c754ec2d50b2 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Sat, 19 Jan 2013 12:07:34 +0000 Subject: [PATCH 07/16] Ensure Content-Length is a string. --- requests/models.py | 4 ++-- test_requests.py | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/requests/models.py b/requests/models.py index 5202e6f4..68ee4421 100644 --- a/requests/models.py +++ b/requests/models.py @@ -121,7 +121,7 @@ class RequestEncodingMixin(object): fp = StringIO(fp) if isinstance(fp, bytes): fp = BytesIO(fp) - + if ft: new_v = (fn, fp.read(), ft) else: @@ -346,7 +346,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): ]) try: - length = super_len(data) + length = str(super_len(data)) except (TypeError, AttributeError): length = False diff --git a/test_requests.py b/test_requests.py index 4a709827..3562318f 100644 --- a/test_requests.py +++ b/test_requests.py @@ -9,6 +9,12 @@ import unittest import requests from requests.auth import HTTPDigestAuth +from requests.compat import str + +try: + import StringIO +except ImportError: + import io as StringIO HTTPBIN = os.environ.get('HTTPBIN_URL', 'http://httpbin.org/') @@ -131,8 +137,6 @@ class RequestsTestCase(unittest.TestCase): self.assertEqual(r.status_code, 200) def test_BASICAUTH_TUPLE_HTTP_200_OK_GET(self): - - auth = ('user', 'pass') url = httpbin('basic-auth', 'user', 'pass') @@ -264,6 +268,11 @@ class RequestsTestCase(unittest.TestCase): self.assertEqual(r.status_code, 200) self.assertTrue(b"text/py-content-type" in r.request.body) + def test_content_length_is_string_for_file_objects(self): + r = requests.Request(url='http://httpbin.org/post', + data=StringIO.StringIO('abc')).prepare() + self.assertTrue(type(r.headers['Content-Length']) == str) + if __name__ == '__main__': unittest.main() From e1c4fe21d47df1ec10ef8a46aecdcb0ecfeac67e Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sat, 19 Jan 2013 11:49:52 -0500 Subject: [PATCH 08/16] Fix #1106 --- requests/adapters.py | 3 --- requests/sessions.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 099a0011..c350ae44 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -13,7 +13,6 @@ import socket from .models import Response from .packages.urllib3.poolmanager import PoolManager, ProxyManager from .packages.urllib3.response import HTTPResponse -from .hooks import dispatch_hook from .compat import urlparse, basestring, urldefrag from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers, prepend_scheme_if_needed) @@ -109,8 +108,6 @@ class HTTPAdapter(BaseAdapter): response.request = req response.connection = self - # Run the Response hook. - response = dispatch_hook('response', req.hooks, response) return response def get_connection(self, url, proxies=None): diff --git a/requests/sessions.py b/requests/sessions.py index d65877cd..173a4811 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -295,7 +295,7 @@ class Session(SessionRedirectMixin): resp.history = tuple(history) # Response manipulation hook. - self.response = dispatch_hook('response', hooks, resp) + resp = dispatch_hook('response', hooks, resp) return resp From de32f1774f5e6491aa58075a5e8baad1a85ba330 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Sat, 19 Jan 2013 17:14:32 +0000 Subject: [PATCH 09/16] Remove test that cannot work on Python3. --- test_requests.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test_requests.py b/test_requests.py index 3562318f..425e3570 100644 --- a/test_requests.py +++ b/test_requests.py @@ -268,11 +268,6 @@ class RequestsTestCase(unittest.TestCase): self.assertEqual(r.status_code, 200) self.assertTrue(b"text/py-content-type" in r.request.body) - def test_content_length_is_string_for_file_objects(self): - r = requests.Request(url='http://httpbin.org/post', - data=StringIO.StringIO('abc')).prepare() - self.assertTrue(type(r.headers['Content-Length']) == str) - if __name__ == '__main__': unittest.main() From cf0a5089f585af4e271566d6b0da66be1e9c8ee5 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Mon, 21 Jan 2013 14:19:29 -0500 Subject: [PATCH 10/16] Add py3.3 testing back --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ea263d1d..62a5959a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ python: - 2.6 - 2.7 - 3.2 + - 3.3 env: HTTPBIN_URL=http://httpbin.org/ script: make test install: From c678a7b4026bbb7bebbae836017f67114d463e79 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 21 Jan 2013 19:55:08 +0000 Subject: [PATCH 11/16] Small typo fixes. --- docs/user/advanced.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 51cef7e4..c7000906 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -144,7 +144,7 @@ Streaming Uploads Requests supports streaming uploads, which allow you to send large streams or files without reading them into memory. To stream and upload, simply provide a file-like object for your body:: with open('massive-body') as f: - request.post('http://some.url/streamed', data=f) + requests.post('http://some.url/streamed', data=f) Chunk-Encoded Requests @@ -157,7 +157,7 @@ Requests also supports Chunked transfer encoding for outgoing and incoming reque yield 'hi' yield 'there' - request.post('http://some.url/chunked', data=gen()) + requests.post('http://some.url/chunked', data=gen()) Event Hooks From 297aa04beaee5e80845841f257723fc472fd6fd7 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 21 Jan 2013 21:15:04 +0000 Subject: [PATCH 12/16] Decrease default line length for iter_lines --- requests/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/models.py b/requests/models.py index 5202e6f4..bbef7ad8 100644 --- a/requests/models.py +++ b/requests/models.py @@ -29,7 +29,7 @@ from .compat import ( REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) CONTENT_CHUNK_SIZE = 10 * 1024 -ITER_CHUNK_SIZE = 10 * 1024 +ITER_CHUNK_SIZE = 512 log = logging.getLogger(__name__) @@ -121,7 +121,7 @@ class RequestEncodingMixin(object): fp = StringIO(fp) if isinstance(fp, bytes): fp = BytesIO(fp) - + if ft: new_v = (fn, fp.read(), ft) else: From 27e814ad7682c2bd7ae8af503a9dcca7840cde64 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 21 Jan 2013 20:21:08 -0500 Subject: [PATCH 13/16] Fix failing tests. --- requests/sessions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requests/sessions.py b/requests/sessions.py index 173a4811..4ad8d6ab 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -275,6 +275,10 @@ class Session(SessionRedirectMixin): # Prepare the Request. prep = req.prepare() + # If auth hooks are present, they aren't passed to `dispatch_hook` + # As such, we need to update the original hooks dictionary with them + hooks.update(prep.hooks) + # Send the request. resp = self.send(prep, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) From a6360ca134623a9faecfe64b51e86f42bffb4dd9 Mon Sep 17 00:00:00 2001 From: Juan Riaza Date: Tue, 22 Jan 2013 16:35:16 +0100 Subject: [PATCH 14/16] missing algorithm field --- requests/auth.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/requests/auth.py b/requests/auth.py index 277e6010..bf6465c1 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -68,18 +68,21 @@ class HTTPDigestAuth(AuthBase): realm = self.chal['realm'] nonce = self.chal['nonce'] qop = self.chal.get('qop') - algorithm = self.chal.get('algorithm', 'MD5') - opaque = self.chal.get('opaque', None) + algorithm = self.chal.get('algorithm') + opaque = self.chal.get('opaque') - algorithm = algorithm.upper() + if algorithm is None: + _algorithm = 'MD5' + else: + _algorithm = algorithm.upper() # lambdas assume digest modules are imported at the top level - if algorithm == 'MD5': + if _algorithm == 'MD5': def md5_utf8(x): if isinstance(x, str): x = x.encode('utf-8') return hashlib.md5(x).hexdigest() hash_utf8 = md5_utf8 - elif algorithm == 'SHA': + elif _algorithm == 'SHA': def sha_utf8(x): if isinstance(x, str): x = x.encode('utf-8') @@ -129,9 +132,10 @@ class HTTPDigestAuth(AuthBase): 'response="%s"' % (self.username, realm, nonce, path, respdig) if opaque: base += ', opaque="%s"' % opaque + if algorithm: + base += ', algorithm="%s"' % algorithm if entdig: base += ', digest="%s"' % entdig - base += ', algorithm="%s"' % algorithm if qop: base += ', qop=auth, nc=%s, cnonce="%s"' % (ncvalue, cnonce) From ca85b4ec6ce2428891d2e1ae72dad2b873c87bbe Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Tue, 22 Jan 2013 17:21:34 -0500 Subject: [PATCH 15/16] Tests pass this time. --- requests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 8c7d40dd..104b6db7 100644 --- a/requests/models.py +++ b/requests/models.py @@ -283,7 +283,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): # Support for unicode domain names and paths. scheme, netloc, path, _params, query, fragment = urlparse(url) - if not scheme: + if not (scheme and netloc): raise MissingSchema("Invalid URL %r: No schema supplied" % url) try: From 6e53290acd02085424c548c6e149c3811325d314 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Tue, 22 Jan 2013 14:40:10 -0800 Subject: [PATCH 16/16] update urllib3 to current master with ssl bugfixes --- requests/packages/urllib3/connectionpool.py | 26 +++++------ requests/packages/urllib3/poolmanager.py | 17 +++++-- requests/packages/urllib3/response.py | 12 ++++- requests/packages/urllib3/util.py | 52 ++++++++++++++++++--- 4 files changed, 81 insertions(+), 26 deletions(-) diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index af8760dc..51c87f58 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -9,6 +9,7 @@ import socket import errno from socket import error as SocketError, timeout as SocketTimeout +from .util import resolve_cert_reqs, resolve_ssl_version try: # Python 3 from http.client import HTTPConnection, HTTPException @@ -80,31 +81,29 @@ class VerifiedHTTPSConnection(HTTPSConnection): ssl_version = None def set_cert(self, key_file=None, cert_file=None, - cert_reqs='CERT_NONE', ca_certs=None): - ssl_req_scheme = { - 'CERT_NONE': ssl.CERT_NONE, - 'CERT_OPTIONAL': ssl.CERT_OPTIONAL, - 'CERT_REQUIRED': ssl.CERT_REQUIRED - } + cert_reqs=None, ca_certs=None): self.key_file = key_file self.cert_file = cert_file - self.cert_reqs = ssl_req_scheme.get(cert_reqs) or ssl.CERT_NONE + self.cert_reqs = cert_reqs self.ca_certs = ca_certs def connect(self): # Add certificate verification sock = socket.create_connection((self.host, self.port), self.timeout) + resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs) + resolved_ssl_version = resolve_ssl_version(self.ssl_version) + # Wrap socket using verification with the root certs in # trusted_root_certs self.sock = ssl_wrap_socket(sock, self.key_file, self.cert_file, - cert_reqs=self.cert_reqs, + cert_reqs=resolved_cert_reqs, ca_certs=self.ca_certs, server_hostname=self.host, - ssl_version=self.ssl_version) + ssl_version=resolved_ssl_version) - if self.ca_certs: + if resolved_cert_reqs != ssl.CERT_NONE: match_hostname(self.sock.getpeercert(), self.host) @@ -514,7 +513,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): strict=False, timeout=None, maxsize=1, block=False, headers=None, key_file=None, cert_file=None, - cert_reqs='CERT_NONE', ca_certs=None, ssl_version=None): + cert_reqs=None, ca_certs=None, ssl_version=None): HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize, @@ -548,10 +547,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): connection.set_cert(key_file=self.key_file, cert_file=self.cert_file, cert_reqs=self.cert_reqs, ca_certs=self.ca_certs) - if self.ssl_version is None: - connection.ssl_version = ssl.PROTOCOL_SSLv23 - else: - connection.ssl_version = self.ssl_version + connection.ssl_version = self.ssl_version return connection diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index a1242026..d58f9f71 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -58,6 +58,17 @@ class PoolManager(RequestMethods): self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close()) + def _new_pool(self, scheme, host, port): + """ + Create a new :class:`ConnectionPool` based on host, port and scheme. + + This method is used to actually create the connection pools handed out + by :meth:`connection_from_url` and companion methods. It is intended + to be overridden for customization. + """ + pool_cls = pool_classes_by_scheme[scheme] + return pool_cls(host, port, **self.connection_pool_kw) + def clear(self): """ Empty our store of pools and direct them all to close. @@ -74,6 +85,7 @@ class PoolManager(RequestMethods): If ``port`` isn't given, it will be derived from the ``scheme`` using ``urllib3.connectionpool.port_by_scheme``. """ + scheme = scheme or 'http' port = port or port_by_scheme.get(scheme, 80) pool_key = (scheme, host, port) @@ -85,11 +97,8 @@ class PoolManager(RequestMethods): return pool # Make a fresh ConnectionPool of the desired type - pool_cls = pool_classes_by_scheme[scheme] - pool = pool_cls(host, port, **self.connection_pool_kw) - + pool = self._new_pool(scheme, host, port) self.pools[pool_key] = pool - return pool def connection_from_url(self, url): diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 833be62b..0761dc03 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -145,7 +145,17 @@ class HTTPResponse(object): # cStringIO doesn't like amt=None data = self._fp.read() else: - return self._fp.read(amt) + data = self._fp.read(amt) + if amt != 0 and not data: # Platform-specific: Buggy versions of Python. + # Close the connection when no data is returned + # + # This is redundant to what httplib/http.client _should_ + # already do. However, versions of python released before + # December 15, 2012 (http://bugs.python.org/issue16298) do not + # properly close the connection in all cases. There is no harm + # in redundantly calling close. + self._fp.close() + return data try: if decode_content and decoder: diff --git a/requests/packages/urllib3/util.py b/requests/packages/urllib3/util.py index 8d8654fe..b827bc4f 100644 --- a/requests/packages/urllib3/util.py +++ b/requests/packages/urllib3/util.py @@ -22,6 +22,7 @@ try: # Test for SSL features SSLContext = None HAS_SNI = False + import ssl from ssl import wrap_socket, CERT_NONE, SSLError, PROTOCOL_SSLv23 from ssl import SSLContext # Modern SSL? from ssl import HAS_SNI # Has SNI? @@ -263,10 +264,48 @@ def is_connection_dropped(conn): return True +def resolve_cert_reqs(candidate): + """ + Resolves the argument to a numeric constant, which can be passed to + the wrap_socket function/method from the ssl module. + Defaults to :data:`ssl.CERT_NONE`. + If given a string it is assumed to be the name of the constant in the + :mod:`ssl` module or its abbrevation. + (So you can specify `REQUIRED` instead of `CERT_REQUIRED`. + If it's neither `None` nor a string we assume it is already the numeric + constant which can directly be passed to wrap_socket. + """ + if candidate is None: + return CERT_NONE + + if isinstance(candidate, str): + res = getattr(ssl, candidate, None) + if res is None: + res = getattr(ssl, 'CERT_' + candidate) + return res + + return candidate + + +def resolve_ssl_version(candidate): + """ + like resolve_cert_reqs + """ + if candidate is None: + return PROTOCOL_SSLv23 + + if isinstance(candidate, str): + res = getattr(ssl, candidate, None) + if res is None: + res = getattr(ssl, 'PROTOCOL_' + candidate) + return res + + return candidate + if SSLContext is not None: # Python 3.2+ - def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=CERT_NONE, + def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ca_certs=None, server_hostname=None, - ssl_version=PROTOCOL_SSLv23): + ssl_version=None): """ All arguments except `server_hostname` have the same meaning as for :func:`ssl.wrap_socket` @@ -279,8 +318,9 @@ if SSLContext is not None: # Python 3.2+ if ca_certs: try: context.load_verify_locations(ca_certs) - except TypeError as e: # Reraise as SSLError - # FIXME: This block needs a test. + # Py32 raises IOError + # Py33 raises FileNotFoundError + except Exception as e: # Reraise as SSLError raise SSLError(e) if certfile: # FIXME: This block needs a test. @@ -290,9 +330,9 @@ if SSLContext is not None: # Python 3.2+ return context.wrap_socket(sock) else: # Python 3.1 and earlier - def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=CERT_NONE, + def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ca_certs=None, server_hostname=None, - ssl_version=PROTOCOL_SSLv23): + ssl_version=None): return wrap_socket(sock, keyfile=keyfile, certfile=certfile, ca_certs=ca_certs, cert_reqs=cert_reqs, ssl_version=ssl_version)