Merge remote-tracking branch 'origin/master'

.gitignore

@@ -5,6 +5,7 @@ nosetests.xml
 junit-report.xml
 pylint.txt
 toy.py
+tox.ini
 violations.pyflakes.txt
 cover/
 build/

AUTHORS.rst

@@ -124,3 +124,7 @@ Patches and Suggestions
 - Wilfred Hughes <me@wilfred.me.uk> @dontYetKnow
 - Dmitry Medvinsky <me@dmedvinsky.name>
 - Bryce Boe <bbzbryce@gmail.com> @bboe
+- Colin Dunklau <colin.dunklau@gmail.com> @cdunklau
+- Hugo Osvaldo Barrera <hugo@osvaldobarrera.com.ar> @hobarrera
+- Łukasz Langa <lukasz@langa.pl> @llanga
+- Dave Shawley <daveshawley@gmail.com>

README.rst

@@ -72,6 +72,11 @@ Or, if you absolutely must:
 But, you really shouldn't do that.
 
 
+Documentation
+-------------
+
+Documentation is available at http://docs.python-requests.org/.
+
 
 Contribute
 ----------

docs/api.rst

@@ -48,13 +48,11 @@ Request Sessions
 Exceptions
 ~~~~~~~~~~
 
-.. module:: requests
-
-.. autoexception:: RequestException
-.. autoexception:: ConnectionError
-.. autoexception:: HTTPError
-.. autoexception:: URLRequired
-.. autoexception:: TooManyRedirects
+.. autoexception:: requests.exceptions.RequestException
+.. autoexception:: requests.exceptions.ConnectionError
+.. autoexception:: requests.exceptions.HTTPError
+.. autoexception:: requests.exceptions.URLRequired
+.. autoexception:: requests.exceptions.TooManyRedirects
 
 
 Status Code Lookup
@@ -76,18 +74,17 @@ Status Code Lookup
 Cookies
 ~~~~~~~
 
-.. autofunction:: dict_from_cookiejar
-.. autofunction:: cookiejar_from_dict
-.. autofunction:: add_dict_to_cookiejar
+.. autofunction:: requests.utils.dict_from_cookiejar
+.. autofunction:: requests.utils.cookiejar_from_dict
+.. autofunction:: requests.utils.add_dict_to_cookiejar
 
 
 Encodings
 ~~~~~~~~~
 
-.. autofunction:: get_encodings_from_content
-.. autofunction:: get_encoding_from_headers
-.. autofunction:: get_unicode_from_response
-.. autofunction:: decode_gzip
+.. autofunction:: requests.utils.get_encodings_from_content
+.. autofunction:: requests.utils.get_encoding_from_headers
+.. autofunction:: requests.utils.get_unicode_from_response
 
 
 Classes

docs/community/support.rst

@@ -3,7 +3,7 @@
 Support
 =======
 
-If you have a questions or issues about Requests, there are several options:
+If you have questions or issues about Requests, there are several options:
 
 Send a Tweet
 ------------

docs/user/authentication.rst

@@ -49,7 +49,7 @@ OAuth 1 Authentication
 
 A common form of authentication for several web APIs is OAuth. The ``requests-oauthlib`` library allows Requests users to easily make OAuth authenticated requests::
 
-    >>> import request
+    >>> import requests
     >>> from requests_oauthlib import OAuth1
 
     >>> url = 'https://api.twitter.com/1.1/account/verify_credentials.json'

requests/__init__.py

@@ -48,6 +48,12 @@ __author__ = 'Kenneth Reitz'
 __license__ = 'Apache 2.0'
 __copyright__ = 'Copyright 2013 Kenneth Reitz'
 
+# Attempt to enable urllib3's SNI support, if possible
+try:
+    from requests.packages.urllib3.contrib import pyopenssl
+    pyopenssl.inject_into_urllib3()
+except ImportError:
+    pass
 
 from . import utils
 from .models import Request, Response, PreparedRequest

requests/models.py

@@ -18,6 +18,7 @@ from .structures import CaseInsensitiveDict
 from .auth import HTTPBasicAuth
 from .cookies import cookiejar_from_dict, get_cookie_header
 from .packages.urllib3.filepost import encode_multipart_formdata
+from .packages.urllib3.util import parse_url
 from .exceptions import HTTPError, RequestException, MissingSchema, InvalidURL
 from .utils import (
     guess_filename, get_auth_from_url, requote_uri,
@@ -284,19 +285,28 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
             pass
 
         # Support for unicode domain names and paths.
-        scheme, netloc, path, _params, query, fragment = urlparse(url)
+        scheme, auth, host, port, path, query, fragment = parse_url(url)
 
         if not scheme:
             raise MissingSchema("Invalid URL %r: No schema supplied" % url)
 
-        if not netloc:
-            raise InvalidURL("Invalid URL %t: No netloc supplied" % url)
+        if not host:
+            raise InvalidURL("Invalid URL %t: No host supplied" % url)
 
+        # Only want to apply IDNA to the hostname
         try:
-            netloc = netloc.encode('idna').decode('utf-8')
+            host = host.encode('idna').decode('utf-8')
         except UnicodeError:
             raise InvalidURL('URL has an invalid label.')
 
+        # Carefully reconstruct the network location
+        netloc = auth or ''
+        if netloc:
+            netloc += '@'
+        netloc += host
+        if port:
+            netloc += ':' + str(port)
+
         # Bare domains aren't valid URLs.
         if not path:
             path = '/'
@@ -308,8 +318,6 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
                 netloc = netloc.encode('utf-8')
             if isinstance(path, str):
                 path = path.encode('utf-8')
-            if isinstance(_params, str):
-                _params = _params.encode('utf-8')
             if isinstance(query, str):
                 query = query.encode('utf-8')
             if isinstance(fragment, str):
@@ -322,7 +330,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
             else:
                 query = enc_params
 
-        url = requote_uri(urlunparse([scheme, netloc, path, _params, query, fragment]))
+        url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment]))
         self.url = url
 
     def prepare_headers(self, headers):
@@ -646,7 +654,7 @@ class Response(object):
     def links(self):
         """Returns the parsed header links of the response, if any."""
 
-        header = self.headers['link']
+        header = self.headers.get('link')
 
         # l = MultiDict()
         l = {}

requests/sessions.py

@@ -11,14 +11,13 @@ requests (cookies, auth, proxies).
 import os
 from datetime import datetime
 
-from .compat import cookielib
+from .compat import cookielib, OrderedDict, urljoin, urlparse
 from .cookies import cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar
 from .models import Request, PreparedRequest
 from .hooks import default_hooks, dispatch_hook
 from .utils import from_key_val_list, default_headers
 from .exceptions import TooManyRedirects, InvalidSchema
 
-from .compat import urlparse, urljoin
 from .adapters import HTTPAdapter
 
 from .utils import requote_uri, get_environ_proxies, get_netrc_auth
@@ -223,9 +222,9 @@ class Session(SessionRedirectMixin):
         self.cookies = cookiejar_from_dict({})
 
         # Default connection adapters.
-        self.adapters = {}
-        self.mount('http://', HTTPAdapter())
+        self.adapters = OrderedDict()
         self.mount('https://', HTTPAdapter())
+        self.mount('http://', HTTPAdapter())
 
     def __enter__(self):
         return self
@@ -490,8 +489,13 @@ class Session(SessionRedirectMixin):
             v.close()
 
     def mount(self, prefix, adapter):
-        """Registers a connection adapter to a prefix."""
+        """Registers a connection adapter to a prefix.
+
+        Adapters are sorted in descending order by key length."""
         self.adapters[prefix] = adapter
+        keys_to_move = [k for k in self.adapters if len(k) < len(prefix)]
+        for key in keys_to_move:
+            self.adapters[key] = self.adapters.pop(key)
 
     def __getstate__(self):
         return dict((attr, getattr(self, attr, None)) for attr in self.__attrs__)

requests/structures.py

@@ -9,6 +9,7 @@ Data structures that power Requests.
 """
 
 import os
+import collections
 from itertools import islice
 
 
@@ -33,43 +34,79 @@ class IteratorProxy(object):
         return "".join(islice(self.i, None, n))
 
 
-class CaseInsensitiveDict(dict):
-    """Case-insensitive Dictionary
+class CaseInsensitiveDict(collections.MutableMapping):
+    """
+    A case-insensitive ``dict``-like object.
+
+    Implements all methods and operations of
+    ``collections.MutableMapping`` as well as dict's ``copy``. Also
+    provides ``lower_items``.
+
+    All keys are expected to be strings. The structure remembers the
+    case of the last key to be set, and ``iter(instance)``,
+    ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()``
+    will contain case-sensitive keys. However, querying and contains
+    testing is case insensitive:
+
+        cid = CaseInsensitiveDict()
+        cid['Accept'] = 'application/json'
+        cid['aCCEPT'] == 'application/json'  # True
+        list(cid) == ['Accept']  # True
 
     For example, ``headers['content-encoding']`` will return the
-    value of a ``'Content-Encoding'`` response header."""
+    value of a ``'Content-Encoding'`` response header, regardless
+    of how the header name was originally stored.
 
-    @property
-    def lower_keys(self):
-        if not hasattr(self, '_lower_keys') or not self._lower_keys:
-            self._lower_keys = dict((k.lower(), k) for k in list(self.keys()))
-        return self._lower_keys
+    If the constructor, ``.update``, or equality comparison
+    operations are given keys that have equal ``.lower()``s, the
+    behavior is undefined.
 
-    def _clear_lower_keys(self):
-        if hasattr(self, '_lower_keys'):
-            self._lower_keys.clear()
+    """
+    def __init__(self, data=None, **kwargs):
+        self._store = dict()
+        if data is None:
+            data = {}
+        self.update(data, **kwargs)
 
     def __setitem__(self, key, value):
-        dict.__setitem__(self, key, value)
-        self._clear_lower_keys()
-
-    def __delitem__(self, key):
-        dict.__delitem__(self, self.lower_keys.get(key.lower(), key))
-        self._lower_keys.clear()
-
-    def __contains__(self, key):
-        return key.lower() in self.lower_keys
+        # Use the lowercased key for lookups, but store the actual
+        # key alongside the value.
+        self._store[key.lower()] = (key, value)
 
     def __getitem__(self, key):
-        # We allow fall-through here, so values default to None
-        if key in self:
-            return dict.__getitem__(self, self.lower_keys[key.lower()])
+        return self._store[key.lower()][1]
 
-    def get(self, key, default=None):
-        if key in self:
-            return self[key]
+    def __delitem__(self, key):
+        del self._store[key.lower()]
+
+    def __iter__(self):
+        return (casedkey for casedkey, mappedvalue in self._store.values())
+
+    def __len__(self):
+        return len(self._store)
+
+    def lower_items(self):
+        """Like iteritems(), but with all lowercase keys."""
+        return (
+            (lowerkey, keyval[1])
+            for (lowerkey, keyval)
+            in self._store.items()
+        )
+
+    def __eq__(self, other):
+        if isinstance(other, collections.Mapping):
+            other = CaseInsensitiveDict(other)
         else:
-            return default
+            return NotImplemented
+        # Compare insensitively
+        return dict(self.lower_items()) == dict(other.lower_items())
+
+    # Copy is required
+    def copy(self):
+         return CaseInsensitiveDict(self._store.values())
+
+    def __repr__(self):
+        return '%s(%r)' % (self.__class__.__name__, dict(self.items()))
 
 
 class LookupDict(dict):

requests/utils.py

@@ -23,6 +23,7 @@ from . import certs
 from .compat import parse_http_list as _parse_list_header
 from .compat import quote, urlparse, bytes, str, OrderedDict, urlunparse
 from .cookies import RequestsCookieJar, cookiejar_from_dict
+from .structures import CaseInsensitiveDict
 
 _hush_pyflakes = (RequestsCookieJar,)
 
@@ -449,11 +450,11 @@ def default_user_agent():
 
 
 def default_headers():
-    return {
+    return CaseInsensitiveDict({
         'User-Agent': default_user_agent(),
         'Accept-Encoding': ', '.join(('gzip', 'deflate', 'compress')),
         'Accept': '*/*'
-    }
+    })
 
 
 def parse_header_links(value):

setup.py

@@ -20,6 +20,7 @@ packages = [
     'requests.packages.charade',
     'requests.packages.urllib3',
     'requests.packages.urllib3.packages',
+    'requests.packages.urllib3.contrib',
     'requests.packages.urllib3.packages.ssl_match_hostname'
 ]
 
@@ -51,9 +52,7 @@ setup(
         'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        # 'Programming Language :: Python :: 3.0',
-        'Programming Language :: Python :: 3.1',
-        'Programming Language :: Python :: 3.2',
         'Programming Language :: Python :: 3.3',
+
     ),
 )

test_requests.py

@@ -11,8 +11,10 @@ import pickle
 
 import requests
 from requests.auth import HTTPDigestAuth
+from requests.adapters import HTTPAdapter
 from requests.compat import str, cookielib
 from requests.cookies import cookiejar_from_dict
+from requests.structures import CaseInsensitiveDict
 
 try:
     import StringIO
@@ -458,6 +460,220 @@ class RequestsTestCase(unittest.TestCase):
         r = s.send(r.prepare())
         self.assertEqual(r.status_code, 200)
 
+    def test_fixes_1329(self):
+        """
+        Ensure that header updates are done case-insensitively.
+        """
+        s = requests.Session()
+        s.headers.update({'ACCEPT': 'BOGUS'})
+        s.headers.update({'accept': 'application/json'})
+        r = s.get(httpbin('get'))
+        headers = r.request.headers
+        # ASCII encode because of key comparison changes in py3
+        self.assertEqual(
+            headers['accept'.encode('ascii')],
+            'application/json'
+        )
+        self.assertEqual(
+            headers['Accept'.encode('ascii')],
+            'application/json'
+        )
+        self.assertEqual(
+            headers['ACCEPT'.encode('ascii')],
+            'application/json'
+        )
+
+    def test_transport_adapter_ordering(self):
+        s = requests.Session()
+        order = ['https://', 'http://']
+        self.assertEqual(order, list(s.adapters))
+        s.mount('http://git', HTTPAdapter())
+        s.mount('http://github', HTTPAdapter())
+        s.mount('http://github.com', HTTPAdapter())
+        s.mount('http://github.com/about/', HTTPAdapter())
+        order = [
+            'http://github.com/about/',
+            'http://github.com',
+            'http://github',
+            'http://git',
+            'https://',
+            'http://',
+        ]
+        self.assertEqual(order, list(s.adapters))
+        s.mount('http://gittip', HTTPAdapter())
+        s.mount('http://gittip.com', HTTPAdapter())
+        s.mount('http://gittip.com/about/', HTTPAdapter())
+        order = [
+            'http://github.com/about/',
+            'http://gittip.com/about/',
+            'http://github.com',
+            'http://gittip.com',
+            'http://github',
+            'http://gittip',
+            'http://git',
+            'https://',
+            'http://',
+        ]
+        self.assertEqual(order, list(s.adapters))
+        s2 = requests.Session()
+        s2.adapters = {'http://': HTTPAdapter()}
+        s2.mount('https://', HTTPAdapter())
+        self.assertTrue('http://' in s2.adapters)
+        self.assertTrue('https://' in s2.adapters)
+
+    def test_long_authinfo_in_url(self):
+        url = 'http://{0}:{1}@{2}:9000/path?query#frag'.format(
+            'E8A3BE87-9E3F-4620-8858-95478E385B5B',
+            'EA770032-DA4D-4D84-8CE9-29C6D910BF1E',
+            'exactly-------------sixty-----------three------------characters',
+        )
+        r = requests.Request('GET', url).prepare()
+        self.assertEqual(r.url, url)
+
+
+class TestCaseInsensitiveDict(unittest.TestCase):
+
+    def test_mapping_init(self):
+        cid = CaseInsensitiveDict({'Foo': 'foo','BAr': 'bar'})
+        self.assertEqual(len(cid), 2)
+        self.assertTrue('foo' in cid)
+        self.assertTrue('bar' in cid)
+
+    def test_iterable_init(self):
+        cid = CaseInsensitiveDict([('Foo', 'foo'), ('BAr', 'bar')])
+        self.assertEqual(len(cid), 2)
+        self.assertTrue('foo' in cid)
+        self.assertTrue('bar' in cid)
+
+    def test_kwargs_init(self):
+        cid = CaseInsensitiveDict(FOO='foo', BAr='bar')
+        self.assertEqual(len(cid), 2)
+        self.assertTrue('foo' in cid)
+        self.assertTrue('bar' in cid)
+
+    def test_docstring_example(self):
+        cid = CaseInsensitiveDict()
+        cid['Accept'] = 'application/json'
+        self.assertEqual(cid['aCCEPT'], 'application/json')
+        self.assertEqual(list(cid), ['Accept'])
+
+    def test_len(self):
+        cid = CaseInsensitiveDict({'a': 'a', 'b': 'b'})
+        cid['A'] = 'a'
+        self.assertEqual(len(cid), 2)
+
+    def test_getitem(self):
+        cid = CaseInsensitiveDict({'Spam': 'blueval'})
+        self.assertEqual(cid['spam'], 'blueval')
+        self.assertEqual(cid['SPAM'], 'blueval')
+
+    def test_fixes_649(self):
+        """__setitem__ should behave case-insensitively."""
+        cid = CaseInsensitiveDict()
+        cid['spam'] = 'oneval'
+        cid['Spam'] = 'twoval'
+        cid['sPAM'] = 'redval'
+        cid['SPAM'] = 'blueval'
+        self.assertEqual(cid['spam'], 'blueval')
+        self.assertEqual(cid['SPAM'], 'blueval')
+        self.assertEqual(list(cid.keys()), ['SPAM'])
+
+    def test_delitem(self):
+        cid = CaseInsensitiveDict()
+        cid['Spam'] = 'someval'
+        del cid['sPam']
+        self.assertFalse('spam' in cid)
+        self.assertEqual(len(cid), 0)
+
+    def test_contains(self):
+        cid = CaseInsensitiveDict()
+        cid['Spam'] = 'someval'
+        self.assertTrue('Spam' in cid)
+        self.assertTrue('spam' in cid)
+        self.assertTrue('SPAM' in cid)
+        self.assertTrue('sPam' in cid)
+        self.assertFalse('notspam' in cid)
+
+    def test_get(self):
+        cid = CaseInsensitiveDict()
+        cid['spam'] = 'oneval'
+        cid['SPAM'] = 'blueval'
+        self.assertEqual(cid.get('spam'), 'blueval')
+        self.assertEqual(cid.get('SPAM'), 'blueval')
+        self.assertEqual(cid.get('sPam'), 'blueval')
+        self.assertEqual(cid.get('notspam', 'default'), 'default')
+
+    def test_update(self):
+        cid = CaseInsensitiveDict()
+        cid['spam'] = 'blueval'
+        cid.update({'sPam': 'notblueval'})
+        self.assertEqual(cid['spam'], 'notblueval')
+        cid = CaseInsensitiveDict({'Foo': 'foo','BAr': 'bar'})
+        cid.update({'fOO': 'anotherfoo', 'bAR': 'anotherbar'})
+        self.assertEqual(len(cid), 2)
+        self.assertEqual(cid['foo'], 'anotherfoo')
+        self.assertEqual(cid['bar'], 'anotherbar')
+
+    def test_update_retains_unchanged(self):
+        cid = CaseInsensitiveDict({'foo': 'foo', 'bar': 'bar'})
+        cid.update({'foo': 'newfoo'})
+        self.assertEquals(cid['bar'], 'bar')
+
+    def test_iter(self):
+        cid = CaseInsensitiveDict({'Spam': 'spam', 'Eggs': 'eggs'})
+        keys = frozenset(['Spam', 'Eggs'])
+        self.assertEqual(frozenset(iter(cid)), keys)
+
+    def test_equality(self):
+        cid = CaseInsensitiveDict({'SPAM': 'blueval', 'Eggs': 'redval'})
+        othercid = CaseInsensitiveDict({'spam': 'blueval', 'eggs': 'redval'})
+        self.assertEqual(cid, othercid)
+        del othercid['spam']
+        self.assertNotEqual(cid, othercid)
+        self.assertEqual(cid, {'spam': 'blueval', 'eggs': 'redval'})
+
+    def test_setdefault(self):
+        cid = CaseInsensitiveDict({'Spam': 'blueval'})
+        self.assertEqual(
+            cid.setdefault('spam', 'notblueval'),
+            'blueval'
+        )
+        self.assertEqual(
+            cid.setdefault('notspam', 'notblueval'),
+            'notblueval'
+        )
+
+    def test_lower_items(self):
+        cid = CaseInsensitiveDict({
+            'Accept': 'application/json',
+            'user-Agent': 'requests',
+        })
+        keyset = frozenset(lowerkey for lowerkey, v in cid.lower_items())
+        lowerkeyset = frozenset(['accept', 'user-agent'])
+        self.assertEqual(keyset, lowerkeyset)
+
+    def test_preserve_key_case(self):
+        cid = CaseInsensitiveDict({
+            'Accept': 'application/json',
+            'user-Agent': 'requests',
+        })
+        keyset = frozenset(['Accept', 'user-Agent'])
+        self.assertEqual(frozenset(i[0] for i in cid.items()), keyset)
+        self.assertEqual(frozenset(cid.keys()), keyset)
+        self.assertEqual(frozenset(cid), keyset)
+
+    def test_preserve_last_key_case(self):
+        cid = CaseInsensitiveDict({
+            'Accept': 'application/json',
+            'user-Agent': 'requests',
+        })
+        cid.update({'ACCEPT': 'application/json'})
+        cid['USER-AGENT'] = 'requests'
+        keyset = frozenset(['ACCEPT', 'USER-AGENT'])
+        self.assertEqual(frozenset(i[0] for i in cid.items()), keyset)
+        self.assertEqual(frozenset(cid.keys()), keyset)
+        self.assertEqual(frozenset(cid), keyset)
+
 
 if __name__ == '__main__':
     unittest.main()