Merge remote-tracking branch 'origin/develop' into develop

AUTHORS.rst

@@ -113,3 +113,4 @@ Patches and Suggestions
 - Jakub Roztocil <jakub@roztocil.name>
 - Ian Cordasco <graffatcolmingov@gmail.com> @sigmavirus24
 - Rhys Elsmore
+- André Graf (dergraf)

docs/user/advanced.rst

@@ -283,7 +283,7 @@ To use the Twitter Streaming API to track the keyword "requests"::
     import json
 
     r = requests.post('https://stream.twitter.com/1/statuses/filter.json',
-        data={'track': 'requests'}, auth=('username', 'password'))
+        data={'track': 'requests'}, auth=('username', 'password'), prefetch=False)
 
     for line in r.iter_lines():
         if line: # filter out keep-alive new lines

docs/user/intro.rst

@@ -17,6 +17,7 @@ Requests was developed with a few :pep:`20` idioms in mind.
 
 All contributions to Requests should keep these important rules in mind.
 
+.. _`isc`:
 
 ISC License
 -----------
@@ -41,9 +42,4 @@ Requests is released under terms of `The ISC License`_.
 Requests License
 ----------------
 
-    Copyright (c) 2011, Kenneth Reitz
-
-    Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
-
-    THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-
+    .. include:: ../../LICENSE

requests/compat.py

@@ -93,6 +93,7 @@ if is_py2:
         import cchardet as chardet
     except ImportError:
         from .packages import chardet
+    from .packages.urllib3.packages.ordered_dict import OrderedDict
 
     builtin_str = str
     bytes = str
@@ -109,6 +110,7 @@ elif is_py3:
     from http.cookies import Morsel
     from io import StringIO
     from .packages import chardet2 as chardet
+    from collections import OrderedDict
 
     builtin_str = str
     str = str

requests/defaults.py

@@ -20,7 +20,7 @@ Configurations:
 :pool_connections: The number of active HTTP connection pools to use.
 :encode_uri: If true, URIs will automatically be percent-encoded.
 :trust_env: If true, the surrouding environment will be trusted (environ, netrc).
-:param store_cookies: If false, the received cookies as part of the HTTP response would be ignored.
+:store_cookies: If false, the received cookies as part of the HTTP response would be ignored.
 
 """
 

requests/models.py

@@ -34,7 +34,7 @@ from .utils import (
     to_key_val_list, DEFAULT_CA_BUNDLE_PATH, parse_header_links, iter_slices)
 from .compat import (
     cookielib, urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes,
-    StringIO, is_py2, chardet, json, builtin_str, numeric_types)
+    StringIO, is_py2, chardet, json, builtin_str)
 
 REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
 CONTENT_CHUNK_SIZE = 10 * 1024
@@ -111,6 +111,10 @@ class Request(object):
         # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'})
         self.proxies = dict(proxies or [])
 
+        for proxy_type,uri_ref in list(self.proxies.items()):
+            if not uri_ref:
+                del self.proxies[proxy_type]
+
         # If no proxies are given, allow configuration by environment variables
         # HTTP_PROXY and HTTPS_PROXY.
         if not self.proxies and self.config.get('trust_env'):
@@ -193,7 +197,7 @@ class Request(object):
                 response.status_code = getattr(resp, 'status', None)
 
                 # Make headers case-insensitive.
-                response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None))
+                response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {}))
 
                 # Set encoding.
                 response.encoding = get_encoding_from_headers(response.headers)
@@ -499,6 +503,21 @@ class Request(object):
                 datetime.now().isoformat(), self.method, url
             ))
 
+        # Use .netrc auth if none was provided.
+        if not self.auth and self.config.get('trust_env'):
+            self.auth = get_netrc_auth(url)
+
+        if self.auth:
+            if isinstance(self.auth, tuple) and len(self.auth) == 2:
+                # special-case basic HTTP auth
+                self.auth = HTTPBasicAuth(*self.auth)
+
+            # Allow auth to make its changes.
+            r = self.auth(self)
+
+            # Update self to reflect the auth changes.
+            self.__dict__.update(r.__dict__)
+
         # Nottin' on you.
         body = None
         content_type = None
@@ -519,21 +538,6 @@ class Request(object):
         if (content_type) and (not 'content-type' in self.headers):
             self.headers['Content-Type'] = content_type
 
-        # Use .netrc auth if none was provided.
-        if not self.auth and self.config.get('trust_env'):
-            self.auth = get_netrc_auth(url)
-
-        if self.auth:
-            if isinstance(self.auth, tuple) and len(self.auth) == 2:
-                # special-case basic HTTP auth
-                self.auth = HTTPBasicAuth(*self.auth)
-
-            # Allow auth to make its changes.
-            r = self.auth(self)
-
-            # Update self to reflect the auth changes.
-            self.__dict__.update(r.__dict__)
-
         _p = urlparse(url)
         no_proxy = filter(lambda x: x.strip(), self.proxies.get('no', '').split(','))
         proxy = self.proxies.get(_p.scheme)

requests/sessions.py

@@ -15,7 +15,7 @@ from .cookies import cookiejar_from_dict, remove_cookie_by_name
 from .defaults import defaults
 from .models import Request
 from .hooks import dispatch_hook
-from .utils import header_expand, to_key_val_list
+from .utils import header_expand, from_key_val_list
 from .packages.urllib3.poolmanager import PoolManager
 
 
@@ -34,27 +34,19 @@ def merge_kwargs(local_kwarg, default_kwarg):
     if local_kwarg is None:
         return default_kwarg
 
-    kwargs = default_kwarg
-    # If default_kwargs is a list rather than a dictionary attempt to convert
-    # to dictionary.  If the check fails, return local_kwargs.
-    if isinstance(default_kwarg, list):
-        try:
-            kwargs = dict(kwargs)
-        except ValueError:
-            return local_kwarg
-
     # Bypass if not a dictionary (e.g. timeout)
-    if not hasattr(kwargs, 'items'):
+    if not hasattr(default_kwarg, 'items'):
         return local_kwarg
 
-    local_kwarg = to_key_val_list(local_kwarg)
+    default_kwarg = from_key_val_list(default_kwarg)
+    local_kwarg = from_key_val_list(local_kwarg)
 
     # Update new values.
-    kwargs = kwargs.copy()
+    kwargs = default_kwarg.copy()
     kwargs.update(local_kwarg)
 
     # Remove keys that are set to None.
-    for (k, v) in local_kwarg:
+    for (k, v) in local_kwarg.items():
         if v is None:
             del kwargs[k]
 
@@ -81,14 +73,13 @@ class Session(object):
         verify=True,
         cert=None):
 
-        #self.headers = to_key_val_list(headers or [])
-        self.headers = headers or {}
+        self.headers = from_key_val_list(headers or [])
         self.auth = auth
         self.timeout = timeout
-        self.proxies = to_key_val_list(proxies or [])
-        self.hooks = hooks or {}
-        self.params = to_key_val_list(params or [])
-        self.config = config or {}
+        self.proxies = from_key_val_list(proxies or [])
+        self.hooks = from_key_val_list(hooks or {})
+        self.params = from_key_val_list(params or [])
+        self.config = from_key_val_list(config or {})
         self.prefetch = prefetch
         self.verify = verify
         self.cert = cert
@@ -171,7 +162,7 @@ class Session(object):
         data = [] if data is None else data
         files = [] if files is None else files
         headers = {} if headers is None else headers
-        params = [] if params is None else params
+        params = {} if params is None else params
         hooks = {} if hooks is None else hooks
         prefetch = prefetch if prefetch is not None else self.prefetch
 
@@ -181,25 +172,23 @@ class Session(object):
 
         # Expand header values.
         if headers:
-            #e = [(k, header_expand(v)) for k, v in to_key_val_list(headers)]
-            #headers = e
-            for k, v in list(headers.items()) or {}:
+            for k, v in list(headers.items() or {}):
                 headers[k] = header_expand(v)
 
         args = dict(
             method=method,
             url=url,
             data=data,
-            params=params,
-            headers=headers,
+            params=from_key_val_list(params),
+            headers=from_key_val_list(headers),
             cookies=cookies,
             files=files,
             auth=auth,
-            hooks=hooks,
+            hooks=from_key_val_list(hooks),
             timeout=timeout,
             allow_redirects=allow_redirects,
-            proxies=to_key_val_list(proxies),
-            config=config,
+            proxies=from_key_val_list(proxies),
+            config=from_key_val_list(config),
             prefetch=prefetch,
             verify=verify,
             cert=cert,

requests/structures.py

@@ -8,6 +8,7 @@ Data structures that power Requests.
 
 """
 
+
 class CaseInsensitiveDict(dict):
     """Case-insensitive Dictionary
 

requests/utils.py

@@ -20,7 +20,7 @@ from netrc import netrc, NetrcParseError
 
 from . import __version__
 from .compat import parse_http_list as _parse_list_header
-from .compat import quote, urlparse, basestring, bytes, str
+from .compat import quote, urlparse, basestring, bytes, str, OrderedDict
 from .cookies import RequestsCookieJar, cookiejar_from_dict
 
 _hush_pyflakes = (RequestsCookieJar,)
@@ -114,28 +114,49 @@ def guess_filename(obj):
         return name
 
 
-def to_key_val_list(value):
+def from_key_val_list(value):
     """Take an object and test to see if it can be represented as a
-    dictionary. Unless it can not be represented as such, return a list of
-    tuples, e.g.,:
+    dictionary. Unless it can not be represented as such, return an
+    OrderedDict, e.g.,
 
-    >>> to_key_val_list([('key', 'val')])
-    [('key', 'val')]
-    >>> to_key_val_list('string')
-    ValueError: ...
-    >>> to_key_val_list({'key': 'val'})
-    [('key', 'val')]
+    ::
+
+        >>> from_key_val_list([('key', 'val')])
+        OrderedDict([('key', 'val')])
+        >>> from_key_val_list('string')
+        ValueError: need more than 1 value to unpack
+        >>> from_key_val_list({'key': 'val'})
+        OrderedDict([('key', 'val')])
     """
     if value is None:
         return None
 
-    try:
-        dict(value)
-    except ValueError:
-        raise ValueError('Unable to encode lists with elements that are not '
-                '2-tuples.')
+    if isinstance(value, (str, bytes, bool, int)):
+        raise ValueError('cannot encode objects that are not 2-tuples')
 
-    if isinstance(value, dict) or hasattr(value, 'items'):
+    return OrderedDict(value)
+
+
+def to_key_val_list(value):
+    """Take an object and test to see if it can be represented as a
+    dictionary. If it can be, return a list of tuples, e.g.,
+
+    ::
+
+        >>> to_key_val_list([('key', 'val')])
+        [('key', 'val')]
+        >>> to_key_val_list({'key': 'val'})
+        [('key', 'val')]
+        >>> to_key_val_list('string')
+        ValueError: cannot encode objects that are not 2-tuples.
+    """
+    if value is None:
+        return None
+
+    if isinstance(value, (str, bytes, bool, int)):
+        raise ValueError('cannot encode objects that are not 2-tuples')
+
+    if isinstance(value, dict):
         value = value.items()
 
     return list(value)
@@ -531,7 +552,7 @@ def parse_header_links(value):
     i.e. Link: <http:/.../front.jpeg>; rel=front; type="image/jpeg",<http://.../back.jpeg>; rel=back;type="image/jpeg"
 
     """
-    
+
     links = []
 
     replace_chars = " '\""
@@ -551,7 +572,7 @@ def parse_header_links(value):
                 key,value = param.split("=")
             except ValueError:
                 break
-            
+
             link[key.strip(replace_chars)] = value.strip(replace_chars)
 
         links.append(link)

setup.py

@@ -67,3 +67,5 @@ setup(
         'Programming Language :: Python :: 3.1',
     ),
 )
+
+del os.environ['PYTHONDONTWRITEBYTECODE']

tests/test_proxies.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys, os, unittest
+
+# Path hack.
+sys.path.insert(0, os.path.abspath('..'))
+import requests
+
+
+class HTTPSProxyTest(unittest.TestCase):
+    """Smoke test for https functionality."""
+
+    smoke_url = "https://github.com"
+
+    def test_empty_https_proxy(self):
+        proxy = {"https" : "" }
+        result = requests.get(self.smoke_url, verify=False, proxies = proxy)
+        self.assertEqual(result.status_code, 200)
+
+    def test_empty_http_proxy(self):
+        proxy = {"http" : "" }
+        result = requests.get(self.smoke_url, proxies = proxy)
+        self.assertEqual(result.status_code, 200)
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_requests.py

@@ -1058,9 +1058,9 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
     def test_prefetch_redirect_bug(self):
         """Test that prefetch persists across redirections."""
         res = get(httpbin('redirect/2'), prefetch=False)
-        # prefetch should persist across the redirect; if it doesn't,
-        # this attempt to iterate will crash because the content has already
-        # been read.
+        # prefetch should persist across the redirect;
+        # the content should not have been consumed
+        self.assertFalse(res._content_consumed)
         first_line = next(res.iter_lines())
         self.assertTrue(first_line.strip().decode('utf-8').startswith('{'))
 
@@ -1068,7 +1068,8 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase):
         """Test that prefetch can be overridden as a kwarg to `send`."""
         req = requests.get(httpbin('get'), return_response=False)
         req.send(prefetch=False)
-        # content should not have been prefetched, and iter_lines should succeed
+        # content should not have been prefetched
+        self.assertFalse(req.response._content_consumed)
         first_line = next(req.response.iter_lines())
         self.assertTrue(first_line.strip().decode('utf-8').startswith('{'))