From 724fd44b975c5a078a186e79417656772a3f8ecc Mon Sep 17 00:00:00 2001 From: schlamar Date: Wed, 3 May 2017 16:05:48 +0200 Subject: [PATCH 1/4] revert 8e6e47af and c121b98c --- requests/structures.py | 89 ---------------------------------------- requests/utils.py | 14 +------ tests/test_structures.py | 74 +-------------------------------- 3 files changed, 3 insertions(+), 174 deletions(-) diff --git a/requests/structures.py b/requests/structures.py index 0b1bd1e7..05d2b3f5 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -8,12 +8,9 @@ Data structures that power Requests. """ import collections -import time from .compat import OrderedDict -current_time = getattr(time, 'monotonic', time.time) - class CaseInsensitiveDict(collections.MutableMapping): """A case-insensitive ``dict``-like object. @@ -106,89 +103,3 @@ class LookupDict(dict): def get(self, key, default=None): return self.__dict__.get(key, default) - - -class TimedCacheManaged(object): - """ - Wrap a function call in a timed cache - """ - def __init__(self, fnc): - self.fnc = fnc - self.cache = TimedCache() - - def __call__(self, *args, **kwargs): - key = args[0] - found = None - try: - found = self.cache[key] - except KeyError: - found = self.fnc(key, **kwargs) - self.cache[key] = found - - return found - - -class TimedCache(collections.MutableMapping): - """ - Evicts entries after expiration_secs. If none are expired and maxlen is hit, - will evict the oldest cached entry - """ - def __init__(self, maxlen=32, expiration_secs=60): - """ - :param maxlen: most number of entries to hold on to - :param expiration_secs: the number of seconds to hold on - to entries - """ - self.maxlen = maxlen - self.expiration_secs = expiration_secs - self._dict = OrderedDict() - - def __repr__(self): - return '' % \ - (self.maxlen, len(self._dict), self.expiration_secs) - - def __iter__(self): - return ((key, value[1]) for key, value in self._dict.items()) - - def __delitem__(self, item): - del self._dict[item] - - def __getitem__(self, key): - """ - Look up an item in the cache. If the item - has already expired, it will be invalidated and not returned - - :param key: which entry to look up - :return: the value in the cache, or None - """ - occurred, value = self._dict[key] - now = int(current_time()) - - if now - occurred > self.expiration_secs: - del self._dict[key] - raise KeyError(key) - else: - return value - - def __setitem__(self, key, value): - """ - Locates the value at lookup key, if cache is full, will evict the - oldest entry - - :param key: the key to search the cache for - :param value: the value to be added to the cache - """ - now = int(current_time()) - - while len(self._dict) >= self.maxlen: - self._dict.popitem(last=False) - - self._dict[key] = (now, value) - - def __len__(self): - """:return: the length of the cache""" - return len(self._dict) - - def clear(self): - """Clears the cache""" - return self._dict.clear() diff --git a/requests/utils.py b/requests/utils.py index b89de420..6f783cf1 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -28,7 +28,7 @@ from .compat import ( quote, urlparse, bytes, str, OrderedDict, unquote, getproxies, proxy_bypass, urlunparse, basestring, integer_types) from .cookies import cookiejar_from_dict -from .structures import CaseInsensitiveDict, TimedCacheManaged +from .structures import CaseInsensitiveDict from .exceptions import ( InvalidURL, InvalidHeader, FileModeWarning, UnrewindableBodyError) @@ -577,16 +577,6 @@ def set_environ(env_name, value): os.environ[env_name] = old_value -@TimedCacheManaged -def _proxy_bypass_cached(netloc): - """ - Looks for netloc in the cache, if not found, will call proxy_bypass - for the netloc and store its result in the cache - - :rtype: bool - """ - return proxy_bypass(netloc) - def should_bypass_proxies(url, no_proxy): """ Returns whether we should bypass proxies or not. @@ -634,7 +624,7 @@ def should_bypass_proxies(url, no_proxy): # legitimate problems. with set_environ('no_proxy', no_proxy_arg): try: - bypass = _proxy_bypass_cached(netloc) + bypass = proxy_bypass(netloc) except (TypeError, socket.gaierror): bypass = False diff --git a/tests/test_structures.py b/tests/test_structures.py index a28e041e..e4d2459f 100644 --- a/tests/test_structures.py +++ b/tests/test_structures.py @@ -2,7 +2,7 @@ import pytest -from requests.structures import CaseInsensitiveDict, LookupDict, TimedCache, TimedCacheManaged +from requests.structures import CaseInsensitiveDict, LookupDict class TestCaseInsensitiveDict: @@ -74,75 +74,3 @@ class TestLookupDict: @get_item_parameters def test_get(self, key, value): assert self.lookup_dict.get(key) == value - - -class TestTimedCache(object): - @pytest.fixture(autouse=True) - def setup(self): - self.any_value = 'some value' - self.expiration_secs = 60 - self.cache = TimedCache(expiration_secs=self.expiration_secs) - yield - self.cache.clear() - - def test_get(self): - self.cache['a'] = self.any_value - assert self.cache['a'] is self.any_value - - def test_repr(self): - repr = str(self.cache) - assert repr == '' - - def test_get_expired_item(self, mocker): - self.cache = TimedCache(maxlen=1, expiration_secs=self.expiration_secs) - - mocker.patch('requests.structures.current_time', lambda: 0) - self.cache['a'] = self.any_value - mocker.patch('requests.structures.current_time', lambda: self.expiration_secs + 1) - assert self.cache.get('a') is None - - def test_evict_first_entry_when_full(self, mocker): - self.cache = TimedCache(maxlen=2, expiration_secs=2) - mocker.patch('requests.structures.current_time', lambda: 0) - self.cache['a'] = self.any_value - mocker.patch('requests.structures.current_time', lambda: 1) - self.cache['b'] = self.any_value - mocker.patch('requests.structures.current_time', lambda: 3) - self.cache['c'] = self.any_value - assert len(self.cache) is 2 - with pytest.raises(KeyError, message='Expected key not found'): - self.cache['a'] - assert self.cache['b'] is self.any_value - assert self.cache['c'] is self.any_value - - def test_delete_item_removes_item(self): - self.cache['a'] = self.any_value - del self.cache['a'] - with pytest.raises(KeyError, message='Expected key not found'): - self.cache['a'] - - def test_iterating_hides_timestamps(self): - self.cache['a'] = 1 - self.cache['b'] = 2 - expected = [('a', 1), ('b', 2)] - actual = [(key, val) for key, val in self.cache] - assert expected == actual - - -class TestTimedCacheManagedDecorator(object): - def test_caches_repeated_calls(self, mocker): - mocker.patch('requests.structures.current_time', lambda: 0) - - nonlocals = {'value': 0} - - @TimedCacheManaged - def some_method(x): - nonlocals['value'] = nonlocals['value'] + x - return nonlocals['value'] - - first_result = some_method(1) - assert first_result is 1 - second_result = some_method(1) - assert second_result is 1 - third_result = some_method(2) - assert third_result is 3 From 4f34446b363d3dec84eedb410df79e5e3773cde9 Mon Sep 17 00:00:00 2001 From: schlamar Date: Thu, 20 Apr 2017 15:41:43 +0200 Subject: [PATCH 2/4] test proxy bypass with config from registry --- tests/test_utils.py | 48 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 11ebf617..fa5da071 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import os from io import BytesIO import pytest @@ -599,3 +600,50 @@ def test_should_bypass_proxies_no_proxy( no_proxy = '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1' # Test 'no_proxy' argument assert should_bypass_proxies(url, no_proxy=no_proxy) == expected + + +@pytest.mark.skipif(os.name != 'nt', reason='Test only on Windows') +@pytest.mark.parametrize( + 'url, expected', ( + ('http://192.168.0.1:5000/', True), + ('http://192.168.0.1/', True), + ('http://172.16.1.1/', True), + ('http://172.16.1.1:5000/', True), + ('http://localhost.localdomain:5000/v1.0/', True), + ('http://172.16.1.22/', False), + ('http://172.16.1.22:5000/', False), + ('http://google.com:5000/v1.0/', False), + )) +def test_should_bypass_proxies_win_registry(url, expected, monkeypatch): + """Tests for function should_bypass_proxies to check if proxy + can be bypassed or not with Windows registry settings + """ + if compat.is_py3: + import winreg + else: + import _winreg as winreg + + class RegHandle: + def Close(self): + pass + + ie_settings = RegHandle() + + def OpenKey(key, subkey): + return ie_settings + + def QueryValueEx(key, value_name): + if key is ie_settings: + if value_name == 'ProxyEnable': + return [1] + elif value_name == 'ProxyOverride': + return ['192.168.*;127.0.0.1;localhost.localdomain;172.16.1.1'] + + monkeypatch.setenv('http_proxy', '') + monkeypatch.setenv('https_proxy', '') + monkeypatch.setenv('ftp_proxy', '') + monkeypatch.setenv('no_proxy', '') + monkeypatch.setenv('NO_PROXY', '') + monkeypatch.setattr(winreg, 'OpenKey', OpenKey) + monkeypatch.setattr(winreg, 'QueryValueEx', QueryValueEx) + assert should_bypass_proxies(url, no_proxy=None) == expected From 1c38e1f5f64c5da8a39a890d9205dd49297a9914 Mon Sep 17 00:00:00 2001 From: schlamar Date: Thu, 20 Apr 2017 16:19:58 +0200 Subject: [PATCH 3/4] proxy bypass on Windows without DNS lookups --- requests/compat.py | 6 ++++-- requests/utils.py | 52 ++++++++++++++++++++++++++++++++++++++++++++- tests/test_utils.py | 27 +++++++++++++---------- 3 files changed, 71 insertions(+), 14 deletions(-) diff --git a/requests/compat.py b/requests/compat.py index f88e600d..a6452c87 100644 --- a/requests/compat.py +++ b/requests/compat.py @@ -37,7 +37,9 @@ except (ImportError, SyntaxError): # --------- if is_py2: - from urllib import quote, unquote, quote_plus, unquote_plus, urlencode, getproxies, proxy_bypass + from urllib import ( + quote, unquote, quote_plus, unquote_plus, urlencode, getproxies, + proxy_bypass, proxy_bypass_environment, getproxies_environment) from urlparse import urlparse, urlunparse, urljoin, urlsplit, urldefrag from urllib2 import parse_http_list import cookielib @@ -54,7 +56,7 @@ if is_py2: elif is_py3: from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, quote_plus, unquote_plus, urldefrag - from urllib.request import parse_http_list, getproxies, proxy_bypass + from urllib.request import parse_http_list, getproxies, proxy_bypass, proxy_bypass_environment, getproxies_environment from http import cookiejar as cookielib from http.cookies import Morsel from io import StringIO diff --git a/requests/utils.py b/requests/utils.py index 6f783cf1..6feaaf38 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -14,6 +14,7 @@ import collections import contextlib import io import os +import platform import re import socket import struct @@ -26,7 +27,8 @@ from ._internal_utils import to_native_string from .compat import parse_http_list as _parse_list_header from .compat import ( quote, urlparse, bytes, str, OrderedDict, unquote, getproxies, - proxy_bypass, urlunparse, basestring, integer_types) + proxy_bypass, urlunparse, basestring, integer_types, is_py3, + proxy_bypass_environment, getproxies_environment) from .cookies import cookiejar_from_dict from .structures import CaseInsensitiveDict from .exceptions import ( @@ -37,6 +39,54 @@ NETRC_FILES = ('.netrc', '_netrc') DEFAULT_CA_BUNDLE_PATH = certs.where() +if platform.system() == 'Windows': + # provide a proxy_bypass version on Windows without DNS lookups + + def proxy_bypass_registry(host): + if is_py3: + import winreg + else: + import _winreg as winreg + try: + internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, + r'Software\Microsoft\Windows\CurrentVersion\Internet Settings') + proxyEnable = winreg.QueryValueEx(internetSettings, + 'ProxyEnable')[0] + proxyOverride = winreg.QueryValueEx(internetSettings, + 'ProxyOverride')[0] + except OSError: + return False + if not proxyEnable or not proxyOverride: + return False + + # make a check value list from the registry entry: replace the + # '' string by the localhost entry and the corresponding + # canonical entry. + proxyOverride = proxyOverride.split(';') + # now check if we match one of the registry values. + for test in proxyOverride: + if test == '': + if '.' not in host: + return True + test = test.replace(".", r"\.") # mask dots + test = test.replace("*", r".*") # change glob sequence + test = test.replace("?", r".") # change glob char + if re.match(test, host, re.I): + return True + return False + + def proxy_bypass(host): # noqa + """Return True, if the host should be bypassed. + + Checks proxy settings gathered from the environment, if specified, + or the registry. + """ + if getproxies_environment(): + return proxy_bypass_environment(host) + else: + return proxy_bypass_registry(host) + + def dict_to_sequence(d): """Returns an internal sequence dictionary update.""" diff --git a/tests/test_utils.py b/tests/test_utils.py index fa5da071..0b37e57f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -604,20 +604,25 @@ def test_should_bypass_proxies_no_proxy( @pytest.mark.skipif(os.name != 'nt', reason='Test only on Windows') @pytest.mark.parametrize( - 'url, expected', ( - ('http://192.168.0.1:5000/', True), - ('http://192.168.0.1/', True), - ('http://172.16.1.1/', True), - ('http://172.16.1.1:5000/', True), - ('http://localhost.localdomain:5000/v1.0/', True), - ('http://172.16.1.22/', False), - ('http://172.16.1.22:5000/', False), - ('http://google.com:5000/v1.0/', False), + 'url, expected, override', ( + ('http://192.168.0.1:5000/', True, None), + ('http://192.168.0.1/', True, None), + ('http://172.16.1.1/', True, None), + ('http://172.16.1.1:5000/', True, None), + ('http://localhost.localdomain:5000/v1.0/', True, None), + ('http://172.16.1.22/', False, None), + ('http://172.16.1.22:5000/', False, None), + ('http://google.com:5000/v1.0/', False, None), + ('http://mylocalhostname:5000/v1.0/', True, ''), + ('http://192.168.0.1/', False, ''), )) -def test_should_bypass_proxies_win_registry(url, expected, monkeypatch): +def test_should_bypass_proxies_win_registry(url, expected, override, + monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not with Windows registry settings """ + if override is None: + override = '192.168.*;127.0.0.1;localhost.localdomain;172.16.1.1' if compat.is_py3: import winreg else: @@ -637,7 +642,7 @@ def test_should_bypass_proxies_win_registry(url, expected, monkeypatch): if value_name == 'ProxyEnable': return [1] elif value_name == 'ProxyOverride': - return ['192.168.*;127.0.0.1;localhost.localdomain;172.16.1.1'] + return [override] monkeypatch.setenv('http_proxy', '') monkeypatch.setenv('https_proxy', '') From 5c389cf3f533846456de9f56d1fea5687a0e2671 Mon Sep 17 00:00:00 2001 From: schlamar Date: Thu, 4 May 2017 15:31:54 +0200 Subject: [PATCH 4/4] update changelog --- HISTORY.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index a2268b2c..f6562c90 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,8 @@ Release History resolution on Windows. - Added ``win_inet_pton`` as conditional dependency for the ``[socks]`` extra on Windows with Python 2.7. +- Changed the proxy bypass implementation on Windows: the proxy bypass + check doesn't use forward and reverse DNS requests anymore 2.13.0 (2017-01-24) +++++++++++++++++++