diff --git a/AUTHORS.rst b/AUTHORS.rst index e25efbbc..1d66c4bf 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -113,3 +113,4 @@ Patches and Suggestions - Jakub Roztocil - Ian Cordasco @sigmavirus24 - Rhys Elsmore +- André Graf (dergraf) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index d976503f..f9f314a7 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -283,7 +283,7 @@ To use the Twitter Streaming API to track the keyword "requests":: import json r = requests.post('https://stream.twitter.com/1/statuses/filter.json', - data={'track': 'requests'}, auth=('username', 'password')) + data={'track': 'requests'}, auth=('username', 'password'), prefetch=False) for line in r.iter_lines(): if line: # filter out keep-alive new lines diff --git a/docs/user/intro.rst b/docs/user/intro.rst index 15a17790..c40528de 100644 --- a/docs/user/intro.rst +++ b/docs/user/intro.rst @@ -17,6 +17,7 @@ Requests was developed with a few :pep:`20` idioms in mind. All contributions to Requests should keep these important rules in mind. +.. _`isc`: ISC License ----------- @@ -41,9 +42,4 @@ Requests is released under terms of `The ISC License`_. Requests License ---------------- - Copyright (c) 2011, Kenneth Reitz - - Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - + .. include:: ../../LICENSE diff --git a/requests/compat.py b/requests/compat.py index 1d4f4a92..351b7c6e 100644 --- a/requests/compat.py +++ b/requests/compat.py @@ -93,6 +93,7 @@ if is_py2: import cchardet as chardet except ImportError: from .packages import chardet + from .packages.urllib3.packages.ordered_dict import OrderedDict builtin_str = str bytes = str @@ -109,6 +110,7 @@ elif is_py3: from http.cookies import Morsel from io import StringIO from .packages import chardet2 as chardet + from collections import OrderedDict builtin_str = str str = str diff --git a/requests/defaults.py b/requests/defaults.py index 87e088b7..4e862d67 100644 --- a/requests/defaults.py +++ b/requests/defaults.py @@ -20,7 +20,7 @@ Configurations: :pool_connections: The number of active HTTP connection pools to use. :encode_uri: If true, URIs will automatically be percent-encoded. :trust_env: If true, the surrouding environment will be trusted (environ, netrc). -:param store_cookies: If false, the received cookies as part of the HTTP response would be ignored. +:store_cookies: If false, the received cookies as part of the HTTP response would be ignored. """ diff --git a/requests/models.py b/requests/models.py index 1159ad5f..d8456375 100644 --- a/requests/models.py +++ b/requests/models.py @@ -34,7 +34,7 @@ from .utils import ( to_key_val_list, DEFAULT_CA_BUNDLE_PATH, parse_header_links, iter_slices) from .compat import ( cookielib, urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes, - StringIO, is_py2, chardet, json, builtin_str, numeric_types) + StringIO, is_py2, chardet, json, builtin_str) REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) CONTENT_CHUNK_SIZE = 10 * 1024 @@ -111,6 +111,10 @@ class Request(object): # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) self.proxies = dict(proxies or []) + for proxy_type,uri_ref in list(self.proxies.items()): + if not uri_ref: + del self.proxies[proxy_type] + # If no proxies are given, allow configuration by environment variables # HTTP_PROXY and HTTPS_PROXY. if not self.proxies and self.config.get('trust_env'): @@ -193,7 +197,7 @@ class Request(object): response.status_code = getattr(resp, 'status', None) # Make headers case-insensitive. - response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) + response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {})) # Set encoding. response.encoding = get_encoding_from_headers(response.headers) @@ -499,6 +503,21 @@ class Request(object): datetime.now().isoformat(), self.method, url )) + # Use .netrc auth if none was provided. + if not self.auth and self.config.get('trust_env'): + self.auth = get_netrc_auth(url) + + if self.auth: + if isinstance(self.auth, tuple) and len(self.auth) == 2: + # special-case basic HTTP auth + self.auth = HTTPBasicAuth(*self.auth) + + # Allow auth to make its changes. + r = self.auth(self) + + # Update self to reflect the auth changes. + self.__dict__.update(r.__dict__) + # Nottin' on you. body = None content_type = None @@ -519,21 +538,6 @@ class Request(object): if (content_type) and (not 'content-type' in self.headers): self.headers['Content-Type'] = content_type - # Use .netrc auth if none was provided. - if not self.auth and self.config.get('trust_env'): - self.auth = get_netrc_auth(url) - - if self.auth: - if isinstance(self.auth, tuple) and len(self.auth) == 2: - # special-case basic HTTP auth - self.auth = HTTPBasicAuth(*self.auth) - - # Allow auth to make its changes. - r = self.auth(self) - - # Update self to reflect the auth changes. - self.__dict__.update(r.__dict__) - _p = urlparse(url) no_proxy = filter(lambda x: x.strip(), self.proxies.get('no', '').split(',')) proxy = self.proxies.get(_p.scheme) diff --git a/requests/sessions.py b/requests/sessions.py index 1f466886..f0d4f3c7 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -15,7 +15,7 @@ from .cookies import cookiejar_from_dict, remove_cookie_by_name from .defaults import defaults from .models import Request from .hooks import dispatch_hook -from .utils import header_expand, to_key_val_list +from .utils import header_expand, from_key_val_list from .packages.urllib3.poolmanager import PoolManager @@ -34,27 +34,19 @@ def merge_kwargs(local_kwarg, default_kwarg): if local_kwarg is None: return default_kwarg - kwargs = default_kwarg - # If default_kwargs is a list rather than a dictionary attempt to convert - # to dictionary. If the check fails, return local_kwargs. - if isinstance(default_kwarg, list): - try: - kwargs = dict(kwargs) - except ValueError: - return local_kwarg - # Bypass if not a dictionary (e.g. timeout) - if not hasattr(kwargs, 'items'): + if not hasattr(default_kwarg, 'items'): return local_kwarg - local_kwarg = to_key_val_list(local_kwarg) + default_kwarg = from_key_val_list(default_kwarg) + local_kwarg = from_key_val_list(local_kwarg) # Update new values. - kwargs = kwargs.copy() + kwargs = default_kwarg.copy() kwargs.update(local_kwarg) # Remove keys that are set to None. - for (k, v) in local_kwarg: + for (k, v) in local_kwarg.items(): if v is None: del kwargs[k] @@ -81,14 +73,13 @@ class Session(object): verify=True, cert=None): - #self.headers = to_key_val_list(headers or []) - self.headers = headers or {} + self.headers = from_key_val_list(headers or []) self.auth = auth self.timeout = timeout - self.proxies = to_key_val_list(proxies or []) - self.hooks = hooks or {} - self.params = to_key_val_list(params or []) - self.config = config or {} + self.proxies = from_key_val_list(proxies or []) + self.hooks = from_key_val_list(hooks or {}) + self.params = from_key_val_list(params or []) + self.config = from_key_val_list(config or {}) self.prefetch = prefetch self.verify = verify self.cert = cert @@ -171,7 +162,7 @@ class Session(object): data = [] if data is None else data files = [] if files is None else files headers = {} if headers is None else headers - params = [] if params is None else params + params = {} if params is None else params hooks = {} if hooks is None else hooks prefetch = prefetch if prefetch is not None else self.prefetch @@ -181,25 +172,23 @@ class Session(object): # Expand header values. if headers: - #e = [(k, header_expand(v)) for k, v in to_key_val_list(headers)] - #headers = e - for k, v in list(headers.items()) or {}: + for k, v in list(headers.items() or {}): headers[k] = header_expand(v) args = dict( method=method, url=url, data=data, - params=params, - headers=headers, + params=from_key_val_list(params), + headers=from_key_val_list(headers), cookies=cookies, files=files, auth=auth, - hooks=hooks, + hooks=from_key_val_list(hooks), timeout=timeout, allow_redirects=allow_redirects, - proxies=to_key_val_list(proxies), - config=config, + proxies=from_key_val_list(proxies), + config=from_key_val_list(config), prefetch=prefetch, verify=verify, cert=cert, diff --git a/requests/structures.py b/requests/structures.py index 7969c7e0..3fda9843 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -8,6 +8,7 @@ Data structures that power Requests. """ + class CaseInsensitiveDict(dict): """Case-insensitive Dictionary diff --git a/requests/utils.py b/requests/utils.py index 3639b8f1..eb146000 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -20,7 +20,7 @@ from netrc import netrc, NetrcParseError from . import __version__ from .compat import parse_http_list as _parse_list_header -from .compat import quote, urlparse, basestring, bytes, str +from .compat import quote, urlparse, basestring, bytes, str, OrderedDict from .cookies import RequestsCookieJar, cookiejar_from_dict _hush_pyflakes = (RequestsCookieJar,) @@ -114,28 +114,49 @@ def guess_filename(obj): return name -def to_key_val_list(value): +def from_key_val_list(value): """Take an object and test to see if it can be represented as a - dictionary. Unless it can not be represented as such, return a list of - tuples, e.g.,: + dictionary. Unless it can not be represented as such, return an + OrderedDict, e.g., - >>> to_key_val_list([('key', 'val')]) - [('key', 'val')] - >>> to_key_val_list('string') - ValueError: ... - >>> to_key_val_list({'key': 'val'}) - [('key', 'val')] + :: + + >>> from_key_val_list([('key', 'val')]) + OrderedDict([('key', 'val')]) + >>> from_key_val_list('string') + ValueError: need more than 1 value to unpack + >>> from_key_val_list({'key': 'val'}) + OrderedDict([('key', 'val')]) """ if value is None: return None - try: - dict(value) - except ValueError: - raise ValueError('Unable to encode lists with elements that are not ' - '2-tuples.') + if isinstance(value, (str, bytes, bool, int)): + raise ValueError('cannot encode objects that are not 2-tuples') - if isinstance(value, dict) or hasattr(value, 'items'): + return OrderedDict(value) + + +def to_key_val_list(value): + """Take an object and test to see if it can be represented as a + dictionary. If it can be, return a list of tuples, e.g., + + :: + + >>> to_key_val_list([('key', 'val')]) + [('key', 'val')] + >>> to_key_val_list({'key': 'val'}) + [('key', 'val')] + >>> to_key_val_list('string') + ValueError: cannot encode objects that are not 2-tuples. + """ + if value is None: + return None + + if isinstance(value, (str, bytes, bool, int)): + raise ValueError('cannot encode objects that are not 2-tuples') + + if isinstance(value, dict): value = value.items() return list(value) @@ -531,7 +552,7 @@ def parse_header_links(value): i.e. Link: ; rel=front; type="image/jpeg",; rel=back;type="image/jpeg" """ - + links = [] replace_chars = " '\"" @@ -551,7 +572,7 @@ def parse_header_links(value): key,value = param.split("=") except ValueError: break - + link[key.strip(replace_chars)] = value.strip(replace_chars) links.append(link) diff --git a/setup.py b/setup.py index d254114a..35c87cef 100755 --- a/setup.py +++ b/setup.py @@ -67,3 +67,5 @@ setup( 'Programming Language :: Python :: 3.1', ), ) + +del os.environ['PYTHONDONTWRITEBYTECODE'] diff --git a/tests/test_proxies.py b/tests/test_proxies.py new file mode 100644 index 00000000..8ab124b2 --- /dev/null +++ b/tests/test_proxies.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys, os, unittest + +# Path hack. +sys.path.insert(0, os.path.abspath('..')) +import requests + + +class HTTPSProxyTest(unittest.TestCase): + """Smoke test for https functionality.""" + + smoke_url = "https://github.com" + + def test_empty_https_proxy(self): + proxy = {"https" : "" } + result = requests.get(self.smoke_url, verify=False, proxies = proxy) + self.assertEqual(result.status_code, 200) + + def test_empty_http_proxy(self): + proxy = {"http" : "" } + result = requests.get(self.smoke_url, proxies = proxy) + self.assertEqual(result.status_code, 200) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_requests.py b/tests/test_requests.py index 9f4e819a..f8a31f24 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1058,9 +1058,9 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): def test_prefetch_redirect_bug(self): """Test that prefetch persists across redirections.""" res = get(httpbin('redirect/2'), prefetch=False) - # prefetch should persist across the redirect; if it doesn't, - # this attempt to iterate will crash because the content has already - # been read. + # prefetch should persist across the redirect; + # the content should not have been consumed + self.assertFalse(res._content_consumed) first_line = next(res.iter_lines()) self.assertTrue(first_line.strip().decode('utf-8').startswith('{')) @@ -1068,7 +1068,8 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): """Test that prefetch can be overridden as a kwarg to `send`.""" req = requests.get(httpbin('get'), return_response=False) req.send(prefetch=False) - # content should not have been prefetched, and iter_lines should succeed + # content should not have been prefetched + self.assertFalse(req.response._content_consumed) first_line = next(req.response.iter_lines()) self.assertTrue(first_line.strip().decode('utf-8').startswith('{'))