From 06d9faedc662a3705af0349e12ea27e2b4bbf640 Mon Sep 17 00:00:00 2001 From: Andre Graf Date: Wed, 29 Aug 2012 21:20:11 +0200 Subject: [PATCH 01/13] data encoding must be done after calling an auth provider which might change the data (as auth.OAuth1 does) --- requests/models.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/requests/models.py b/requests/models.py index 29c06c6b..66c50939 100644 --- a/requests/models.py +++ b/requests/models.py @@ -499,6 +499,21 @@ class Request(object): datetime.now().isoformat(), self.method, url )) + # Use .netrc auth if none was provided. + if not self.auth and self.config.get('trust_env'): + self.auth = get_netrc_auth(url) + + if self.auth: + if isinstance(self.auth, tuple) and len(self.auth) == 2: + # special-case basic HTTP auth + self.auth = HTTPBasicAuth(*self.auth) + + # Allow auth to make its changes. + r = self.auth(self) + + # Update self to reflect the auth changes. + self.__dict__.update(r.__dict__) + # Nottin' on you. body = None content_type = None @@ -519,21 +534,6 @@ class Request(object): if (content_type) and (not 'content-type' in self.headers): self.headers['Content-Type'] = content_type - # Use .netrc auth if none was provided. - if not self.auth and self.config.get('trust_env'): - self.auth = get_netrc_auth(url) - - if self.auth: - if isinstance(self.auth, tuple) and len(self.auth) == 2: - # special-case basic HTTP auth - self.auth = HTTPBasicAuth(*self.auth) - - # Allow auth to make its changes. - r = self.auth(self) - - # Update self to reflect the auth changes. - self.__dict__.update(r.__dict__) - _p = urlparse(url) no_proxy = filter(lambda x: x.strip(), self.proxies.get('no', '').split(',')) proxy = self.proxies.get(_p.scheme) From 52d2e2ee4e37d9594ba62fd3caa952d537f699de Mon Sep 17 00:00:00 2001 From: Andre Graf Date: Wed, 29 Aug 2012 21:20:27 +0200 Subject: [PATCH 02/13] added myself --- AUTHORS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index e25efbbc..1d66c4bf 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -113,3 +113,4 @@ Patches and Suggestions - Jakub Roztocil - Ian Cordasco @sigmavirus24 - Rhys Elsmore +- André Graf (dergraf) From 8081d7b15cf893435c19ea1c0d5e4392a60fe18f Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Sun, 2 Sep 2012 23:09:43 -0400 Subject: [PATCH 03/13] Fixes #817. Use dicts and lists where necessary but accept both dicts and lists of 2-tuples everywhere. --- requests/compat.py | 2 ++ requests/models.py | 2 +- requests/sessions.py | 47 +++++++++++++--------------------- requests/structures.py | 1 + requests/utils.py | 57 +++++++++++++++++++++++++++++------------- 5 files changed, 61 insertions(+), 48 deletions(-) diff --git a/requests/compat.py b/requests/compat.py index 1d4f4a92..351b7c6e 100644 --- a/requests/compat.py +++ b/requests/compat.py @@ -93,6 +93,7 @@ if is_py2: import cchardet as chardet except ImportError: from .packages import chardet + from .packages.urllib3.packages.ordered_dict import OrderedDict builtin_str = str bytes = str @@ -109,6 +110,7 @@ elif is_py3: from http.cookies import Morsel from io import StringIO from .packages import chardet2 as chardet + from collections import OrderedDict builtin_str = str str = str diff --git a/requests/models.py b/requests/models.py index 1159ad5f..b02739f2 100644 --- a/requests/models.py +++ b/requests/models.py @@ -34,7 +34,7 @@ from .utils import ( to_key_val_list, DEFAULT_CA_BUNDLE_PATH, parse_header_links, iter_slices) from .compat import ( cookielib, urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes, - StringIO, is_py2, chardet, json, builtin_str, numeric_types) + StringIO, is_py2, chardet, json, builtin_str) REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) CONTENT_CHUNK_SIZE = 10 * 1024 diff --git a/requests/sessions.py b/requests/sessions.py index 1f466886..f0d4f3c7 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -15,7 +15,7 @@ from .cookies import cookiejar_from_dict, remove_cookie_by_name from .defaults import defaults from .models import Request from .hooks import dispatch_hook -from .utils import header_expand, to_key_val_list +from .utils import header_expand, from_key_val_list from .packages.urllib3.poolmanager import PoolManager @@ -34,27 +34,19 @@ def merge_kwargs(local_kwarg, default_kwarg): if local_kwarg is None: return default_kwarg - kwargs = default_kwarg - # If default_kwargs is a list rather than a dictionary attempt to convert - # to dictionary. If the check fails, return local_kwargs. - if isinstance(default_kwarg, list): - try: - kwargs = dict(kwargs) - except ValueError: - return local_kwarg - # Bypass if not a dictionary (e.g. timeout) - if not hasattr(kwargs, 'items'): + if not hasattr(default_kwarg, 'items'): return local_kwarg - local_kwarg = to_key_val_list(local_kwarg) + default_kwarg = from_key_val_list(default_kwarg) + local_kwarg = from_key_val_list(local_kwarg) # Update new values. - kwargs = kwargs.copy() + kwargs = default_kwarg.copy() kwargs.update(local_kwarg) # Remove keys that are set to None. - for (k, v) in local_kwarg: + for (k, v) in local_kwarg.items(): if v is None: del kwargs[k] @@ -81,14 +73,13 @@ class Session(object): verify=True, cert=None): - #self.headers = to_key_val_list(headers or []) - self.headers = headers or {} + self.headers = from_key_val_list(headers or []) self.auth = auth self.timeout = timeout - self.proxies = to_key_val_list(proxies or []) - self.hooks = hooks or {} - self.params = to_key_val_list(params or []) - self.config = config or {} + self.proxies = from_key_val_list(proxies or []) + self.hooks = from_key_val_list(hooks or {}) + self.params = from_key_val_list(params or []) + self.config = from_key_val_list(config or {}) self.prefetch = prefetch self.verify = verify self.cert = cert @@ -171,7 +162,7 @@ class Session(object): data = [] if data is None else data files = [] if files is None else files headers = {} if headers is None else headers - params = [] if params is None else params + params = {} if params is None else params hooks = {} if hooks is None else hooks prefetch = prefetch if prefetch is not None else self.prefetch @@ -181,25 +172,23 @@ class Session(object): # Expand header values. if headers: - #e = [(k, header_expand(v)) for k, v in to_key_val_list(headers)] - #headers = e - for k, v in list(headers.items()) or {}: + for k, v in list(headers.items() or {}): headers[k] = header_expand(v) args = dict( method=method, url=url, data=data, - params=params, - headers=headers, + params=from_key_val_list(params), + headers=from_key_val_list(headers), cookies=cookies, files=files, auth=auth, - hooks=hooks, + hooks=from_key_val_list(hooks), timeout=timeout, allow_redirects=allow_redirects, - proxies=to_key_val_list(proxies), - config=config, + proxies=from_key_val_list(proxies), + config=from_key_val_list(config), prefetch=prefetch, verify=verify, cert=cert, diff --git a/requests/structures.py b/requests/structures.py index 7969c7e0..3fda9843 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -8,6 +8,7 @@ Data structures that power Requests. """ + class CaseInsensitiveDict(dict): """Case-insensitive Dictionary diff --git a/requests/utils.py b/requests/utils.py index 3639b8f1..eb146000 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -20,7 +20,7 @@ from netrc import netrc, NetrcParseError from . import __version__ from .compat import parse_http_list as _parse_list_header -from .compat import quote, urlparse, basestring, bytes, str +from .compat import quote, urlparse, basestring, bytes, str, OrderedDict from .cookies import RequestsCookieJar, cookiejar_from_dict _hush_pyflakes = (RequestsCookieJar,) @@ -114,28 +114,49 @@ def guess_filename(obj): return name -def to_key_val_list(value): +def from_key_val_list(value): """Take an object and test to see if it can be represented as a - dictionary. Unless it can not be represented as such, return a list of - tuples, e.g.,: + dictionary. Unless it can not be represented as such, return an + OrderedDict, e.g., - >>> to_key_val_list([('key', 'val')]) - [('key', 'val')] - >>> to_key_val_list('string') - ValueError: ... - >>> to_key_val_list({'key': 'val'}) - [('key', 'val')] + :: + + >>> from_key_val_list([('key', 'val')]) + OrderedDict([('key', 'val')]) + >>> from_key_val_list('string') + ValueError: need more than 1 value to unpack + >>> from_key_val_list({'key': 'val'}) + OrderedDict([('key', 'val')]) """ if value is None: return None - try: - dict(value) - except ValueError: - raise ValueError('Unable to encode lists with elements that are not ' - '2-tuples.') + if isinstance(value, (str, bytes, bool, int)): + raise ValueError('cannot encode objects that are not 2-tuples') - if isinstance(value, dict) or hasattr(value, 'items'): + return OrderedDict(value) + + +def to_key_val_list(value): + """Take an object and test to see if it can be represented as a + dictionary. If it can be, return a list of tuples, e.g., + + :: + + >>> to_key_val_list([('key', 'val')]) + [('key', 'val')] + >>> to_key_val_list({'key': 'val'}) + [('key', 'val')] + >>> to_key_val_list('string') + ValueError: cannot encode objects that are not 2-tuples. + """ + if value is None: + return None + + if isinstance(value, (str, bytes, bool, int)): + raise ValueError('cannot encode objects that are not 2-tuples') + + if isinstance(value, dict): value = value.items() return list(value) @@ -531,7 +552,7 @@ def parse_header_links(value): i.e. Link: ; rel=front; type="image/jpeg",; rel=back;type="image/jpeg" """ - + links = [] replace_chars = " '\"" @@ -551,7 +572,7 @@ def parse_header_links(value): key,value = param.split("=") except ValueError: break - + link[key.strip(replace_chars)] = value.strip(replace_chars) links.append(link) From d9a85560a2767e1b1420ab455b2eab508bed6666 Mon Sep 17 00:00:00 2001 From: Gareth Lloyd Date: Thu, 6 Sep 2012 15:13:46 +0100 Subject: [PATCH 04/13] add prefetch=False to streaming example --- docs/user/advanced.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index d976503f..f9f314a7 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -283,7 +283,7 @@ To use the Twitter Streaming API to track the keyword "requests":: import json r = requests.post('https://stream.twitter.com/1/statuses/filter.json', - data={'track': 'requests'}, auth=('username', 'password')) + data={'track': 'requests'}, auth=('username', 'password'), prefetch=False) for line in r.iter_lines(): if line: # filter out keep-alive new lines From c73646da0018063f7ddd03514e6cde55e82bb640 Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Thu, 6 Sep 2012 16:31:01 -0700 Subject: [PATCH 05/13] fix some tests to correctly cover the API After #831, the tests added in #764 (which relied on iter_content() crashing if the response was prefetched) no longer tested what they were intended to test. --- tests/test_requests.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_requests.py b/tests/test_requests.py index 9f4e819a..f8a31f24 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1058,9 +1058,9 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): def test_prefetch_redirect_bug(self): """Test that prefetch persists across redirections.""" res = get(httpbin('redirect/2'), prefetch=False) - # prefetch should persist across the redirect; if it doesn't, - # this attempt to iterate will crash because the content has already - # been read. + # prefetch should persist across the redirect; + # the content should not have been consumed + self.assertFalse(res._content_consumed) first_line = next(res.iter_lines()) self.assertTrue(first_line.strip().decode('utf-8').startswith('{')) @@ -1068,7 +1068,8 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): """Test that prefetch can be overridden as a kwarg to `send`.""" req = requests.get(httpbin('get'), return_response=False) req.send(prefetch=False) - # content should not have been prefetched, and iter_lines should succeed + # content should not have been prefetched + self.assertFalse(req.response._content_consumed) first_line = next(req.response.iter_lines()) self.assertTrue(first_line.strip().decode('utf-8').startswith('{')) From 861e63ce8e4a7c4881b3129edea8f23d730cabf1 Mon Sep 17 00:00:00 2001 From: Alan Hamlett Date: Sun, 9 Sep 2012 01:56:32 -0700 Subject: [PATCH 06/13] fixed typo in docs --- requests/defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/defaults.py b/requests/defaults.py index 87e088b7..4e862d67 100644 --- a/requests/defaults.py +++ b/requests/defaults.py @@ -20,7 +20,7 @@ Configurations: :pool_connections: The number of active HTTP connection pools to use. :encode_uri: If true, URIs will automatically be percent-encoded. :trust_env: If true, the surrouding environment will be trusted (environ, netrc). -:param store_cookies: If false, the received cookies as part of the HTTP response would be ignored. +:store_cookies: If false, the received cookies as part of the HTTP response would be ignored. """ From 91a9c397466d22c83688ddd851d7f04d73cd22a8 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Thu, 6 Sep 2012 10:22:40 -0400 Subject: [PATCH 07/13] Reset PYTHONDONTWRITEBYTECODE after installation Fixes #839. --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index d254114a..9430942b 100755 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ if sys.argv[-1] == 'publish': os.system('python setup.py sdist upload') sys.exit() +bytecode = os.getenv('PYTHONDONTWRITEBYTECODE', '0') os.environ['PYTHONDONTWRITEBYTECODE'] = '1' packages = [ @@ -67,3 +68,5 @@ setup( 'Programming Language :: Python :: 3.1', ), ) + +os.environ['PYTHONDONTWRITEBYTECODE'] = bytecode From aed5bf7cc7440ff1676e14e132f9a1064f2694c9 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Mon, 10 Sep 2012 23:09:41 -0400 Subject: [PATCH 08/13] As per the discussion on #841. --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9430942b..35c87cef 100755 --- a/setup.py +++ b/setup.py @@ -15,7 +15,6 @@ if sys.argv[-1] == 'publish': os.system('python setup.py sdist upload') sys.exit() -bytecode = os.getenv('PYTHONDONTWRITEBYTECODE', '0') os.environ['PYTHONDONTWRITEBYTECODE'] = '1' packages = [ @@ -69,4 +68,4 @@ setup( ), ) -os.environ['PYTHONDONTWRITEBYTECODE'] = bytecode +del os.environ['PYTHONDONTWRITEBYTECODE'] From fc1d0fedea5c400c539b660f7bf8301d855a1d63 Mon Sep 17 00:00:00 2001 From: barberj Date: Thu, 6 Sep 2012 21:17:55 -0400 Subject: [PATCH 09/13] None is not iterable --- requests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 031efee3..e63fd2f2 100644 --- a/requests/models.py +++ b/requests/models.py @@ -193,7 +193,7 @@ class Request(object): response.status_code = getattr(resp, 'status', None) # Make headers case-insensitive. - response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) + response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {})) # Set encoding. response.encoding = get_encoding_from_headers(response.headers) From 2dc7db4730fb9763931ad0a7449732d0797959b9 Mon Sep 17 00:00:00 2001 From: Eric Hansen Date: Tue, 11 Sep 2012 15:40:16 -0400 Subject: [PATCH 10/13] Fixed issue with empty proxies being passed --- requests/models.py | 4 ++++ tests/test_proxies.py | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 tests/test_proxies.py diff --git a/requests/models.py b/requests/models.py index 031efee3..0ef30251 100644 --- a/requests/models.py +++ b/requests/models.py @@ -111,6 +111,10 @@ class Request(object): # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) self.proxies = dict(proxies or []) + for proxy_type,uri_ref in self.proxies.items(): + if not uri_ref: + del self.proxies[proxy_type] + # If no proxies are given, allow configuration by environment variables # HTTP_PROXY and HTTPS_PROXY. if not self.proxies and self.config.get('trust_env'): diff --git a/tests/test_proxies.py b/tests/test_proxies.py new file mode 100644 index 00000000..8ab124b2 --- /dev/null +++ b/tests/test_proxies.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys, os, unittest + +# Path hack. +sys.path.insert(0, os.path.abspath('..')) +import requests + + +class HTTPSProxyTest(unittest.TestCase): + """Smoke test for https functionality.""" + + smoke_url = "https://github.com" + + def test_empty_https_proxy(self): + proxy = {"https" : "" } + result = requests.get(self.smoke_url, verify=False, proxies = proxy) + self.assertEqual(result.status_code, 200) + + def test_empty_http_proxy(self): + proxy = {"http" : "" } + result = requests.get(self.smoke_url, proxies = proxy) + self.assertEqual(result.status_code, 200) + +if __name__ == '__main__': + unittest.main() From 65c19bc0e0c44da54ff46d2237264800e2f2238f Mon Sep 17 00:00:00 2001 From: Eric Hansen Date: Wed, 12 Sep 2012 13:49:34 -0400 Subject: [PATCH 11/13] Fix for v3 in deleting empty proxies. --- requests/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/models.py b/requests/models.py index 0ef30251..af364102 100644 --- a/requests/models.py +++ b/requests/models.py @@ -111,7 +111,7 @@ class Request(object): # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) self.proxies = dict(proxies or []) - for proxy_type,uri_ref in self.proxies.items(): + for proxy_type,uri_ref in list(self.proxies.items()): if not uri_ref: del self.proxies[proxy_type] From fe4211d5b5efb585b014845ef4d99d79cb1f316f Mon Sep 17 00:00:00 2001 From: Priit Laes Date: Thu, 13 Sep 2012 09:03:38 +0300 Subject: [PATCH 12/13] Use the include directive instead of rewriting the license in docs. Helps also keeping the copyright years in sync \o/ --- docs/user/intro.rst | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/docs/user/intro.rst b/docs/user/intro.rst index 15a17790..2f66e4e6 100644 --- a/docs/user/intro.rst +++ b/docs/user/intro.rst @@ -41,9 +41,4 @@ Requests is released under terms of `The ISC License`_. Requests License ---------------- - Copyright (c) 2011, Kenneth Reitz - - Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - + .. include:: ../../LICENSE From cb44e165e56fe58c4fb4394f796ab53877b352c5 Mon Sep 17 00:00:00 2001 From: Priit Laes Date: Thu, 13 Sep 2012 09:21:32 +0300 Subject: [PATCH 13/13] Create label for the ISC License, which is referenced from the index page --- docs/user/intro.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/user/intro.rst b/docs/user/intro.rst index 2f66e4e6..c40528de 100644 --- a/docs/user/intro.rst +++ b/docs/user/intro.rst @@ -17,6 +17,7 @@ Requests was developed with a few :pep:`20` idioms in mind. All contributions to Requests should keep these important rules in mind. +.. _`isc`: ISC License -----------