diff --git a/AUTHORS.rst b/AUTHORS.rst index 3c074d1b..e18d5751 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -116,3 +116,4 @@ Patches and Suggestions - André Graf (dergraf) - Stephen Zhuang (everbird) - Martijn Pieters +- Jonatan Heyman diff --git a/docs/dev/todo.rst b/docs/dev/todo.rst index 7aa2cae8..e9930a8e 100644 --- a/docs/dev/todo.rst +++ b/docs/dev/todo.rst @@ -23,7 +23,19 @@ order to run requests' test suite:: $ make $ make test -The ``Makefile`` has various useful targets for testing. +The ``Makefile`` has various useful targets for testing. For example, if you +want to see how your pull request will behave with Travis-CI you would run +``make travis``. + +Versions of Python to Test On +----------------------------- + +Officially (as of 26-Nov-2012), requests supports python 2.6-3.3. In the +future, support for 3.1 and 3.2 may be dropped. In general you will need to +test on at least one python 2 and one python 3 version. You can also set up +Travis CI for your own fork before you submit a pull request so that you are +assured your fork works. To use Travis CI for your fork and other projects see +their `documentation `_. What Needs to be Done --------------------- diff --git a/requests/api.py b/requests/api.py index ded79352..297f4cbf 100644 --- a/requests/api.py +++ b/requests/api.py @@ -12,10 +12,8 @@ This module implements the Requests API. """ from . import sessions -from .safe_mode import catch_exceptions_if_in_safe_mode -@catch_exceptions_if_in_safe_mode def request(method, url, **kwargs): """Constructs and sends a :class:`Request `. Returns :class:`Response ` object. diff --git a/requests/auth.py b/requests/auth.py index 65568f52..b662397e 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -92,14 +92,28 @@ class OAuth1(AuthBase): # Omit body data in the signing and since it will always # be empty (cant add paras to body if multipart) and we wish # to preserve body. - r.url, r.headers, _ = self.client.sign( - unicode(r.full_url), unicode(r.method), None, r.headers) - elif decoded_body is not None and contenttype in (CONTENT_TYPE_FORM_URLENCODED, ''): - # Normal signing - if not contenttype: - r.headers['Content-Type'] = CONTENT_TYPE_FORM_URLENCODED - r.url, r.headers, r.data = self.client.sign( - unicode(r.full_url), unicode(r.method), r.data, r.headers) + r.url, r.headers, _ = self.client.sign(unicode(r.full_url), + unicode(r.method), + None, + r.headers) + elif (decoded_body is not None and + contenttype == CONTENT_TYPE_FORM_URLENCODED): + # If the Content-Type header is urlencoded and there are no + # illegal characters in the body, assume that the content actually + # is urlencoded, and so should be part of the signature. + r.url, r.headers, r.data = self.client.sign(unicode(r.full_url), + unicode(r.method), + r.data, + r.headers) + elif r.data: + # The data we passed was either definitely not urlencoded + # (because extract_params returned nothing) or doesn't have a + # content header that assures us that it is. Assume then that the + # data shouldn't be part of the signature. + r.url, r.headers, _ = self.client.sign(unicode(r.full_url), + unicode(r.method), + None, + r.headers) else: _oauth_signed = False if _oauth_signed: diff --git a/requests/cookies.py b/requests/cookies.py index c3c2debb..245fdd9e 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -32,9 +32,10 @@ class MockRequest(object): def __init__(self, request): self._r = request self._new_headers = {} + self.type = urlparse(self._r.full_url).scheme def get_type(self): - return urlparse(self._r.full_url).scheme + return self.type def get_host(self): return urlparse(self._r.full_url).netloc diff --git a/requests/models.py b/requests/models.py index 58e7f9a7..c184c1cd 100644 --- a/requests/models.py +++ b/requests/models.py @@ -9,6 +9,7 @@ This module contains the primary objects that power Requests. import os import socket +import collections from datetime import datetime from io import BytesIO @@ -119,7 +120,7 @@ class Request(object): # If no proxies are given, allow configuration by environment variables # HTTP_PROXY and HTTPS_PROXY. if not self.proxies and self.config.get('trust_env'): - self.proxies = get_environ_proxies() + self.proxies = get_environ_proxies(self.url) self.data = data self.params = params @@ -467,10 +468,10 @@ class Request(object): def register_hook(self, event, hook): """Properly register a hook.""" - if callable(hook): + if isinstance(hook, collections.Callable): self.hooks[event].append(hook) elif hasattr(hook, '__iter__'): - self.hooks[event].extend(h for h in hook if callable(h)) + self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable)) def deregister_hook(self, event, hook): """Deregister a previously registered hook. @@ -541,6 +542,14 @@ class Request(object): else: content_type = 'application/x-www-form-urlencoded' + self.headers['Content-Length'] = '0' + if hasattr(body, 'seek') and hasattr(body, 'tell'): + body.seek(0, 2) + self.headers['Content-Length'] = str(body.tell()) + body.seek(0, 0) + elif body is not None: + self.headers['Content-Length'] = str(len(body)) + # Add content-type if it wasn't explicitly provided. if (content_type) and (not 'content-type' in self.headers): self.headers['Content-Type'] = content_type diff --git a/requests/safe_mode.py b/requests/safe_mode.py index 0fb8d705..18808d74 100644 --- a/requests/safe_mode.py +++ b/requests/safe_mode.py @@ -18,17 +18,17 @@ import socket def catch_exceptions_if_in_safe_mode(function): - """New implementation of safe_mode. We catch all exceptions at the API level + """New implementation of safe_mode. We catch all exceptions at the Session level and then return a blank Response object with the error field filled. This decorator - wraps request() in api.py. + wraps Session._send_request() in sessions.py. """ - def wrapped(method, url, **kwargs): + def wrapped(*args, **kwargs): # if save_mode, we catch exceptions and fill error field if (kwargs.get('config') and kwargs.get('config').get('safe_mode')) or (kwargs.get('session') and kwargs.get('session').config.get('safe_mode')): try: - return function(method, url, **kwargs) + return function(*args, **kwargs) except (RequestException, ConnectionError, HTTPError, socket.timeout, socket.gaierror) as e: r = Response() @@ -36,5 +36,5 @@ def catch_exceptions_if_in_safe_mode(function): r.raw = HTTPResponse() # otherwise, tests fail r.status_code = 0 # with this status_code, content returns None return r - return function(method, url, **kwargs) + return function(*args, **kwargs) return wrapped diff --git a/requests/sessions.py b/requests/sessions.py index 0962d819..5d67b4d8 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -17,6 +17,7 @@ from .models import Request from .hooks import dispatch_hook from .utils import header_expand, from_key_val_list from .packages.urllib3.poolmanager import PoolManager +from .safe_mode import catch_exceptions_if_in_safe_mode def merge_kwargs(local_kwarg, default_kwarg): @@ -265,7 +266,12 @@ class Session(object): return r # Send the HTTP Request. - r.send(prefetch=prefetch) + return self._send_request(r, **args) + + @catch_exceptions_if_in_safe_mode + def _send_request(self, r, **kwargs): + # Send the request. + r.send(prefetch=kwargs.get("prefetch")) # Return the response. return r.response diff --git a/requests/utils.py b/requests/utils.py index b3d33f4f..91a3b760 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -29,7 +29,9 @@ CERTIFI_BUNDLE_PATH = None try: # see if requests's own CA certificate bundle is installed from . import certs - CERTIFI_BUNDLE_PATH = certs.where() + path = certs.where() + if os.path.exists(path): + CERTIFI_BUNDLE_PATH = certs.where() except ImportError: pass @@ -473,21 +475,18 @@ def unquote_unreserved(uri): """Un-escape any percent-escape sequences in a URI that are unreserved characters. This leaves all reserved, illegal and non-ASCII bytes encoded. """ - try: - parts = uri.split('%') - for i in range(1, len(parts)): - h = parts[i][0:2] - if len(h) == 2 and h.isalnum(): - c = chr(int(h, 16)) - if c in UNRESERVED_SET: - parts[i] = c + parts[i][2:] - else: - parts[i] = '%' + parts[i] + parts = uri.split('%') + for i in range(1, len(parts)): + h = parts[i][0:2] + if len(h) == 2 and h.isalnum(): + c = chr(int(h, 16)) + if c in UNRESERVED_SET: + parts[i] = c + parts[i][2:] else: parts[i] = '%' + parts[i] - return ''.join(parts) - except ValueError: - return uri + else: + parts[i] = '%' + parts[i] + return ''.join(parts) def requote_uri(uri): @@ -502,7 +501,7 @@ def requote_uri(uri): return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~") -def get_environ_proxies(): +def get_environ_proxies(url): """Return a dict of environment proxies.""" proxy_keys = [ @@ -510,11 +509,29 @@ def get_environ_proxies(): 'http', 'https', 'ftp', - 'socks', - 'no' + 'socks' ] get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper()) + + # First check whether no_proxy is defined. If it is, check that the URL + # we're getting isn't in the no_proxy list. + no_proxy = get_proxy('no_proxy') + + if no_proxy: + # We need to check whether we match here. We need to see if we match + # the end of the netloc, both with and without the port. + no_proxy = no_proxy.split(',') + netloc = urlparse(url).netloc + + for host in no_proxy: + if netloc.endswith(host) or netloc.split(':')[0].endswith(host): + # The URL does match something in no_proxy, so we don't want + # to apply the proxies on this URL. + return {} + + # If we get here, we either didn't have no_proxy set or we're not going + # anywhere that no_proxy applies to. proxies = [(key, get_proxy(key + '_proxy')) for key in proxy_keys] return dict([(key, val) for (key, val) in proxies if val]) diff --git a/tests/test_requests.py b/tests/test_requests.py index 6615678f..a4129203 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -11,6 +11,7 @@ import json import unittest import pickle import tempfile +import collections import requests from requests.compat import str, StringIO @@ -805,7 +806,7 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): def assert_hooks_are_callable(hooks): for h in hooks['args']: - self.assertTrue(callable(h)) + self.assertTrue(isinstance(h, collections.Callable)) hooks = [add_foo_header, add_bar_header] r = requests.models.Request() @@ -929,6 +930,19 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): ds2 = pickle.loads(pickle.dumps(requests.session(prefetch=False))) self.assertTrue(ds1.prefetch) self.assertFalse(ds2.prefetch) + + def test_session_connection_error_with_safe_mode(self): + config = {"safe_mode":True} + + s = requests.session() + r = s.get("http://localhost:1/nope", timeout=0.1, config=config) + self.assertFalse(r.ok) + self.assertTrue(r.content is None) + + s2 = requests.session(config=config) + r2 = s2.get("http://localhost:1/nope", timeout=0.1) + self.assertFalse(r2.ok) + self.assertTrue(r2.content is None) def test_connection_error(self): try: diff --git a/tests/test_utils.py b/tests/test_utils.py index 5cd0684e..015cac63 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -9,6 +9,7 @@ import random # Path hack. sys.path.insert(0, os.path.abspath('..')) +from requests.utils import get_environ_proxies import requests.utils from requests.compat import is_py3, bytes @@ -20,7 +21,7 @@ else: byteschr = chr -class GuessJSONUTFTests(unittest.TestCase): +class UtilityTests(unittest.TestCase): """Tests for the JSON UTF encoding guessing code.""" codecs = ( @@ -73,5 +74,49 @@ class GuessJSONUTFTests(unittest.TestCase): continue raise + def test_get_environ_proxies_respects_no_proxy(self): + '''This test confirms that the no_proxy environment setting is + respected by get_environ_proxies().''' + + # Store the current environment settings. + try: + old_http_proxy = os.environ['http_proxy'] + except KeyError: + old_http_proxy = None + + try: + old_no_proxy = os.environ['no_proxy'] + except KeyError: + old_no_proxy = None + + # Set up some example environment settings. + os.environ['http_proxy'] = 'http://www.example.com/' + os.environ['no_proxy'] = r'localhost,.0.0.1:8080' + + # Set up expected proxy return values. + proxy_yes = {'http': 'http://www.example.com/'} + proxy_no = {} + + # Check that we get the right things back. + self.assertEqual(proxy_yes, + get_environ_proxies('http://www.google.com/')) + self.assertEqual(proxy_no, + get_environ_proxies('http://localhost/test')) + self.assertEqual(proxy_no, + get_environ_proxies('http://127.0.0.1:8080/')) + self.assertEqual(proxy_yes, + get_environ_proxies('http://127.0.0.1:8081/')) + + # Return the settings to what they were. + if old_http_proxy: + os.environ['http_proxy'] = old_http_proxy + else: + del os.environ['http_proxy'] + + if old_no_proxy: + os.environ['no_proxy'] = old_no_proxy + else: + del os.environ['no_proxy'] + if __name__ == '__main__': unittest.main()