Merge remote-tracking branch 'origin/develop' into develop

AUTHORS.rst

@@ -116,3 +116,4 @@ Patches and Suggestions
 - André Graf (dergraf)
 - Stephen Zhuang (everbird)
 - Martijn Pieters
+- Jonatan Heyman

docs/dev/todo.rst

@@ -23,7 +23,19 @@ order to run requests' test suite::
     $ make
     $ make test
 
-The ``Makefile`` has various useful targets for testing.
+The ``Makefile`` has various useful targets for testing. For example, if you 
+want to see how your pull request will behave with Travis-CI you would run 
+``make travis``.
+
+Versions of Python to Test On
+-----------------------------
+
+Officially (as of 26-Nov-2012), requests supports python 2.6-3.3. In the 
+future, support for 3.1 and 3.2 may be dropped. In general you will need to 
+test on at least one python 2 and one python 3 version. You can also set up 
+Travis CI for your own fork before you submit a pull request so that you are 
+assured your fork works. To use Travis CI for your fork and other projects see 
+their `documentation <http://about.travis-ci.org/docs/user/getting-started/>`_.
 
 What Needs to be Done
 ---------------------

requests/api.py

@@ -12,10 +12,8 @@ This module implements the Requests API.
 """
 
 from . import sessions
-from .safe_mode import catch_exceptions_if_in_safe_mode
 
 
-@catch_exceptions_if_in_safe_mode
 def request(method, url, **kwargs):
     """Constructs and sends a :class:`Request <Request>`.
     Returns :class:`Response <Response>` object.

requests/auth.py

@@ -92,14 +92,28 @@ class OAuth1(AuthBase):
             # Omit body data in the signing and since it will always
             # be empty (cant add paras to body if multipart) and we wish
             # to preserve body.
-            r.url, r.headers, _ = self.client.sign(
-                unicode(r.full_url), unicode(r.method), None, r.headers)
-        elif decoded_body is not None and contenttype in (CONTENT_TYPE_FORM_URLENCODED, ''):
-            # Normal signing
-            if not contenttype:
-                r.headers['Content-Type'] = CONTENT_TYPE_FORM_URLENCODED
-            r.url, r.headers, r.data = self.client.sign(
-                unicode(r.full_url), unicode(r.method), r.data, r.headers)
+            r.url, r.headers, _ = self.client.sign(unicode(r.full_url),
+                                                   unicode(r.method),
+                                                   None,
+                                                   r.headers)
+        elif (decoded_body is not None and 
+                                  contenttype == CONTENT_TYPE_FORM_URLENCODED):
+            # If the Content-Type header is urlencoded and there are no
+            # illegal characters in the body, assume that the content actually
+            # is urlencoded, and so should be part of the signature.
+            r.url, r.headers, r.data = self.client.sign(unicode(r.full_url),
+                                                        unicode(r.method),
+                                                        r.data,
+                                                        r.headers)
+        elif r.data:
+            # The data we passed was either definitely not urlencoded
+            # (because extract_params returned nothing) or doesn't have a
+            # content header that assures us that it is. Assume then that the
+            # data shouldn't be part of the signature.
+            r.url, r.headers, _ = self.client.sign(unicode(r.full_url),
+                                                   unicode(r.method),
+                                                   None,
+                                                   r.headers)
         else:
             _oauth_signed = False
         if _oauth_signed:

requests/cookies.py

@@ -32,9 +32,10 @@ class MockRequest(object):
     def __init__(self, request):
         self._r = request
         self._new_headers = {}
+        self.type = urlparse(self._r.full_url).scheme
 
     def get_type(self):
-        return urlparse(self._r.full_url).scheme
+        return self.type
 
     def get_host(self):
         return urlparse(self._r.full_url).netloc

requests/models.py

@@ -9,6 +9,7 @@ This module contains the primary objects that power Requests.
 
 import os
 import socket
+import collections
 from datetime import datetime
 from io import BytesIO
 
@@ -119,7 +120,7 @@ class Request(object):
         # If no proxies are given, allow configuration by environment variables
         # HTTP_PROXY and HTTPS_PROXY.
         if not self.proxies and self.config.get('trust_env'):
-            self.proxies = get_environ_proxies()
+            self.proxies = get_environ_proxies(self.url)
 
         self.data = data
         self.params = params
@@ -467,10 +468,10 @@ class Request(object):
 
     def register_hook(self, event, hook):
         """Properly register a hook."""
-        if callable(hook):
+        if isinstance(hook, collections.Callable):
             self.hooks[event].append(hook)
         elif hasattr(hook, '__iter__'):
-            self.hooks[event].extend(h for h in hook if callable(h))
+            self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable))
 
     def deregister_hook(self, event, hook):
         """Deregister a previously registered hook.
@@ -541,6 +542,14 @@ class Request(object):
                 else:
                     content_type = 'application/x-www-form-urlencoded'
 
+        self.headers['Content-Length'] = '0'
+        if hasattr(body, 'seek') and hasattr(body, 'tell'):
+            body.seek(0, 2)
+            self.headers['Content-Length'] = str(body.tell())
+            body.seek(0, 0)
+        elif body is not None:
+            self.headers['Content-Length'] = str(len(body))
+
         # Add content-type if it wasn't explicitly provided.
         if (content_type) and (not 'content-type' in self.headers):
             self.headers['Content-Type'] = content_type

requests/safe_mode.py

@@ -18,17 +18,17 @@ import socket
 
 
 def catch_exceptions_if_in_safe_mode(function):
-    """New implementation of safe_mode. We catch all exceptions at the API level
+    """New implementation of safe_mode. We catch all exceptions at the Session level
     and then return a blank Response object with the error field filled. This decorator
-    wraps request() in api.py.
+    wraps Session._send_request() in sessions.py.
     """
 
-    def wrapped(method, url, **kwargs):
+    def wrapped(*args, **kwargs):
         # if save_mode, we catch exceptions and fill error field
         if (kwargs.get('config') and kwargs.get('config').get('safe_mode')) or (kwargs.get('session')
                                             and kwargs.get('session').config.get('safe_mode')):
             try:
-                return function(method, url, **kwargs)
+                return function(*args, **kwargs)
             except (RequestException, ConnectionError, HTTPError,
                     socket.timeout, socket.gaierror) as e:
                 r = Response()
@@ -36,5 +36,5 @@ def catch_exceptions_if_in_safe_mode(function):
                 r.raw = HTTPResponse()  # otherwise, tests fail
                 r.status_code = 0  # with this status_code, content returns None
                 return r
-        return function(method, url, **kwargs)
+        return function(*args, **kwargs)
     return wrapped

requests/sessions.py

@@ -17,6 +17,7 @@ from .models import Request
 from .hooks import dispatch_hook
 from .utils import header_expand, from_key_val_list
 from .packages.urllib3.poolmanager import PoolManager
+from .safe_mode import catch_exceptions_if_in_safe_mode
 
 
 def merge_kwargs(local_kwarg, default_kwarg):
@@ -265,7 +266,12 @@ class Session(object):
             return r
 
         # Send the HTTP Request.
-        r.send(prefetch=prefetch)
+        return self._send_request(r, **args)
+
+    @catch_exceptions_if_in_safe_mode
+    def _send_request(self, r, **kwargs):
+        # Send the request.
+        r.send(prefetch=kwargs.get("prefetch"))
 
         # Return the response.
         return r.response

requests/utils.py

@@ -29,7 +29,9 @@ CERTIFI_BUNDLE_PATH = None
 try:
     # see if requests's own CA certificate bundle is installed
     from . import certs
-    CERTIFI_BUNDLE_PATH = certs.where()
+    path = certs.where()
+    if os.path.exists(path):
+        CERTIFI_BUNDLE_PATH = certs.where()
 except ImportError:
     pass
 
@@ -473,21 +475,18 @@ def unquote_unreserved(uri):
     """Un-escape any percent-escape sequences in a URI that are unreserved
     characters. This leaves all reserved, illegal and non-ASCII bytes encoded.
     """
-    try:
-        parts = uri.split('%')
-        for i in range(1, len(parts)):
-            h = parts[i][0:2]
-            if len(h) == 2 and h.isalnum():
-                c = chr(int(h, 16))
-                if c in UNRESERVED_SET:
-                    parts[i] = c + parts[i][2:]
-                else:
-                    parts[i] = '%' + parts[i]
+    parts = uri.split('%')
+    for i in range(1, len(parts)):
+        h = parts[i][0:2]
+        if len(h) == 2 and h.isalnum():
+            c = chr(int(h, 16))
+            if c in UNRESERVED_SET:
+                parts[i] = c + parts[i][2:]
             else:
                 parts[i] = '%' + parts[i]
-        return ''.join(parts)
-    except ValueError:
-        return uri
+        else:
+            parts[i] = '%' + parts[i]
+    return ''.join(parts)
 
 
 def requote_uri(uri):
@@ -502,7 +501,7 @@ def requote_uri(uri):
     return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~")
 
 
-def get_environ_proxies():
+def get_environ_proxies(url):
     """Return a dict of environment proxies."""
 
     proxy_keys = [
@@ -510,11 +509,29 @@ def get_environ_proxies():
         'http',
         'https',
         'ftp',
-        'socks',
-        'no'
+        'socks'
     ]
 
     get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper())
+
+    # First check whether no_proxy is defined. If it is, check that the URL
+    # we're getting isn't in the no_proxy list.
+    no_proxy = get_proxy('no_proxy')
+
+    if no_proxy:
+        # We need to check whether we match here. We need to see if we match
+        # the end of the netloc, both with and without the port.
+        no_proxy = no_proxy.split(',')
+        netloc = urlparse(url).netloc
+
+        for host in no_proxy:
+            if netloc.endswith(host) or netloc.split(':')[0].endswith(host):
+                # The URL does match something in no_proxy, so we don't want
+                # to apply the proxies on this URL.
+                return {}
+
+    # If we get here, we either didn't have no_proxy set or we're not going
+    # anywhere that no_proxy applies to.
     proxies = [(key, get_proxy(key + '_proxy')) for key in proxy_keys]
     return dict([(key, val) for (key, val) in proxies if val])
 

tests/test_requests.py

@@ -11,6 +11,7 @@ import json
 import unittest
 import pickle
 import tempfile
+import collections
 
 import requests
 from requests.compat import str, StringIO
@@ -805,7 +806,7 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
 
         def assert_hooks_are_callable(hooks):
             for h in hooks['args']:
-                self.assertTrue(callable(h))
+                self.assertTrue(isinstance(h, collections.Callable))
 
         hooks = [add_foo_header, add_bar_header]
         r = requests.models.Request()
@@ -929,6 +930,19 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
         ds2 = pickle.loads(pickle.dumps(requests.session(prefetch=False)))
         self.assertTrue(ds1.prefetch)
         self.assertFalse(ds2.prefetch)
+    
+    def test_session_connection_error_with_safe_mode(self):
+        config = {"safe_mode":True}
+
+        s = requests.session()
+        r = s.get("http://localhost:1/nope", timeout=0.1, config=config)
+        self.assertFalse(r.ok)
+        self.assertTrue(r.content is None)
+
+        s2 = requests.session(config=config)
+        r2 = s2.get("http://localhost:1/nope", timeout=0.1)
+        self.assertFalse(r2.ok)
+        self.assertTrue(r2.content is None)
 
     def test_connection_error(self):
         try:

tests/test_utils.py

@@ -9,6 +9,7 @@ import random
 
 # Path hack.
 sys.path.insert(0, os.path.abspath('..'))
+from requests.utils import get_environ_proxies
 import requests.utils
 from requests.compat import is_py3, bytes
 
@@ -20,7 +21,7 @@ else:
     byteschr = chr
 
 
-class GuessJSONUTFTests(unittest.TestCase):
+class UtilityTests(unittest.TestCase):
     """Tests for the JSON UTF encoding guessing code."""
 
     codecs = (
@@ -73,5 +74,49 @@ class GuessJSONUTFTests(unittest.TestCase):
                         continue
                     raise
 
+    def test_get_environ_proxies_respects_no_proxy(self):
+        '''This test confirms that the no_proxy environment setting is
+        respected by get_environ_proxies().'''
+
+        # Store the current environment settings.
+        try:
+            old_http_proxy = os.environ['http_proxy']
+        except KeyError:
+            old_http_proxy = None
+
+        try:
+            old_no_proxy = os.environ['no_proxy']
+        except KeyError:
+            old_no_proxy = None
+
+        # Set up some example environment settings.
+        os.environ['http_proxy'] = 'http://www.example.com/'
+        os.environ['no_proxy'] = r'localhost,.0.0.1:8080'
+
+        # Set up expected proxy return values.
+        proxy_yes = {'http': 'http://www.example.com/'}
+        proxy_no = {}
+
+        # Check that we get the right things back.
+        self.assertEqual(proxy_yes,
+                         get_environ_proxies('http://www.google.com/'))
+        self.assertEqual(proxy_no,
+                         get_environ_proxies('http://localhost/test'))
+        self.assertEqual(proxy_no,
+                         get_environ_proxies('http://127.0.0.1:8080/'))
+        self.assertEqual(proxy_yes,
+                         get_environ_proxies('http://127.0.0.1:8081/'))
+
+        # Return the settings to what they were.
+        if old_http_proxy:
+            os.environ['http_proxy'] = old_http_proxy
+        else:
+            del os.environ['http_proxy']
+
+        if old_no_proxy:
+            os.environ['no_proxy'] = old_no_proxy
+        else:
+            del os.environ['no_proxy']
+
 if __name__ == '__main__':
     unittest.main()