From 31c0962e83c8f3b043b3faceff880f69a9a4dbdc Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Sun, 13 Oct 2013 09:53:44 +0100 Subject: [PATCH 01/19] Better connection behaviour for chunked upload. --- requests/adapters.py | 46 ++++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 0adca690..1d246f2d 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -327,27 +327,39 @@ class HTTPAdapter(BaseAdapter): conn = conn.proxy_pool low_conn = conn._get_conn(timeout=timeout) - low_conn.putrequest(request.method, url, skip_accept_encoding=True) - for header, value in request.headers.items(): - low_conn.putheader(header, value) + try: + low_conn.putrequest(request.method, + url, + skip_accept_encoding=True) - low_conn.endheaders() + for header, value in request.headers.items(): + low_conn.putheader(header, value) - for i in request.body: - low_conn.send(hex(len(i))[2:].encode('utf-8')) - low_conn.send(b'\r\n') - low_conn.send(i) - low_conn.send(b'\r\n') - low_conn.send(b'0\r\n\r\n') + low_conn.endheaders() - r = low_conn.getresponse() - resp = HTTPResponse.from_httplib(r, - pool=conn, - connection=low_conn, - preload_content=False, - decode_content=False - ) + for i in request.body: + low_conn.send(hex(len(i))[2:].encode('utf-8')) + low_conn.send(b'\r\n') + low_conn.send(i) + low_conn.send(b'\r\n') + low_conn.send(b'0\r\n\r\n') + + r = low_conn.getresponse() + resp = HTTPResponse.from_httplib(r, + pool=conn, + connection=low_conn, + preload_content=False, + decode_content=False + ) + except: + # If we hit any problems here, clean up the connection. + # Then, reraise so that we can handle the actual exception. + low_conn.close() + raise + else: + # All is well, return the connection to the pool. + conn._put_conn(low_conn) except socket.error as sockerr: raise ConnectionError(sockerr) From 9606f0240b8e0ce5f64e9659c2161a91c8dc6b60 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Sun, 13 Oct 2013 09:54:01 +0100 Subject: [PATCH 02/19] PEP8 fix to make me happier. --- requests/adapters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requests/adapters.py b/requests/adapters.py index 1d246f2d..4a0977a6 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -346,7 +346,8 @@ class HTTPAdapter(BaseAdapter): low_conn.send(b'0\r\n\r\n') r = low_conn.getresponse() - resp = HTTPResponse.from_httplib(r, + resp = HTTPResponse.from_httplib( + r, pool=conn, connection=low_conn, preload_content=False, From 2c914126ba0c4865dc37c0a70f481091112a7dac Mon Sep 17 00:00:00 2001 From: Kamil Madac Date: Mon, 2 Dec 2013 22:32:29 +0100 Subject: [PATCH 03/19] Added possibility to use IP ranges (ex. 192.168.1.0/24) to no_proxy environment variable --- requests/utils.py | 27 ++++++++++++++++++++++----- test_requests.py | 9 +++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 4283560e..f92eed6f 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -17,6 +17,7 @@ import os import platform import re import sys +import netaddr from . import __version__ from . import certs @@ -420,11 +421,27 @@ def get_environ_proxies(url): # the end of the netloc, both with and without the port. no_proxy = no_proxy.replace(' ', '').split(',') - for host in no_proxy: - if netloc.endswith(host) or netloc.split(':')[0].endswith(host): - # The URL does match something in no_proxy, so we don't want - # to apply the proxies on this URL. - return {} + ip = None + try: + ip = netaddr.IPAddress(netloc.split(':')[0]) + except netaddr.AddrFormatError: + pass + + if ip: + for proxy_ip in no_proxy: + proxy_ipaddress = None + try: + proxy_ipaddress = netaddr.IPNetwork(proxy_ip) + except (netaddr.AddrFormatError, ValueError): + continue + if proxy_ipaddress and ip in proxy_ipaddress: + return {} + else: + for host in no_proxy: + if netloc.endswith(host) or netloc.split(':')[0].endswith(host): + # The URL does match something in no_proxy, so we don't want + # to apply the proxies on this URL. + return {} # If the system proxy settings indicate that this URL should be bypassed, # don't proxy. diff --git a/test_requests.py b/test_requests.py index 3b673956..cb91ade1 100755 --- a/test_requests.py +++ b/test_requests.py @@ -910,5 +910,14 @@ class UtilsTestCase(unittest.TestCase): else: assert super_len(cStringIO.StringIO('but some how, some way...')) == 25 + def test_get_environ_proxies_ip_ranges(self): + """ Ensures that IP addresses are correctly matches with ranges in no_proxy variable """ + from requests.utils import get_environ_proxies + os.environ['no_proxy'] = "127.0.0.1,localhost.localdomain,192.168.0.0/24,172.16.1.1" + assert get_environ_proxies('http://192.168.0.1:5000/') == {} + assert get_environ_proxies('http://192.168.0.1/') == {} + assert get_environ_proxies('http://172.16.1.1/') == {} + assert get_environ_proxies('http://192.168.1.1:5000/') == {'no': os.environ['no_proxy']} + if __name__ == '__main__': unittest.main() From 9ec222c363b226e908a9d72018e2f467952da52f Mon Sep 17 00:00:00 2001 From: Kamil Madac Date: Mon, 2 Dec 2013 22:40:26 +0100 Subject: [PATCH 04/19] Added test for domain name no_proxy matching. --- .idea/dictionaries/kmadac.xml | 3 +++ test_requests.py | 8 ++++++++ 2 files changed, 11 insertions(+) create mode 100644 .idea/dictionaries/kmadac.xml diff --git a/.idea/dictionaries/kmadac.xml b/.idea/dictionaries/kmadac.xml new file mode 100644 index 00000000..93bbf4db --- /dev/null +++ b/.idea/dictionaries/kmadac.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/test_requests.py b/test_requests.py index cb91ade1..4aa86c3d 100755 --- a/test_requests.py +++ b/test_requests.py @@ -917,7 +917,15 @@ class UtilsTestCase(unittest.TestCase): assert get_environ_proxies('http://192.168.0.1:5000/') == {} assert get_environ_proxies('http://192.168.0.1/') == {} assert get_environ_proxies('http://172.16.1.1/') == {} + assert get_environ_proxies('http://172.16.1.1:5000/') == {} assert get_environ_proxies('http://192.168.1.1:5000/') == {'no': os.environ['no_proxy']} + def test_get_environ_proxies(self): + """ Ensures that IP addresses are correctly matches with ranges in no_proxy variable """ + from requests.utils import get_environ_proxies + os.environ['no_proxy'] = "127.0.0.1,localhost.localdomain,192.168.0.0/24,172.16.1.1" + assert get_environ_proxies('http://localhost.localdomain:5000/test/') == {} + assert get_environ_proxies('http://www.requests.com/') == {'no': os.environ['no_proxy']} + if __name__ == '__main__': unittest.main() From f9a48e0f0b79c1f6dc148b1abc6a2622c48b1ab1 Mon Sep 17 00:00:00 2001 From: Kamil Madac Date: Mon, 2 Dec 2013 22:42:20 +0100 Subject: [PATCH 05/19] Added kamil.madac to Authors --- AUTHORS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index 2636fa6c..b0a7e241 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -143,3 +143,4 @@ Patches and Suggestions - Thomas Weißschuh @t-8ch - Jayson Vantuyl @kagato - Pengfei.X +- Kamil Madac From 8aff6f5ed0aca2dbe5b2eba242f0e3b96285c700 Mon Sep 17 00:00:00 2001 From: Kamil Madac Date: Tue, 3 Dec 2013 16:23:08 +0100 Subject: [PATCH 06/19] Redesigned no_proxy ip range imlementation to use only stdlib functions --- AUTHORS.rst | 1 + requests/utils.py | 55 ++++++++++++++++++++++++++++++++++------------- test_requests.py | 13 +++++++++-- 3 files changed, 52 insertions(+), 17 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 2636fa6c..b0a7e241 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -143,3 +143,4 @@ Patches and Suggestions - Thomas Weißschuh @t-8ch - Jayson Vantuyl @kagato - Pengfei.X +- Kamil Madac diff --git a/requests/utils.py b/requests/utils.py index f92eed6f..d7852308 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -17,7 +17,8 @@ import os import platform import re import sys -import netaddr +import socket +import struct from . import __version__ from . import certs @@ -406,6 +407,39 @@ def requote_uri(uri): return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~") +def address_in_network(ip,net): + '''This function allows you to check if on IP belogs to a Network''' + ipaddr = struct.unpack('=L', socket.inet_aton(ip))[0] + netaddr, bits = net.split('/') + netmask = struct.unpack('=L', socket.inet_aton(dotted_netmask(int(bits))))[0] + network = struct.unpack('=L', socket.inet_aton(netaddr))[0] & netmask + return (ipaddr & netmask) == (network & netmask) + + +def dotted_netmask(mask): + bits = 0xffffffff ^ (1 << 32 - mask) - 1 + return socket.inet_ntoa(struct.pack('>I', bits)) + + +def is_ipv4_address(string_ip): + try: + socket.inet_aton(string_ip) + except BaseException: + return False + return True + + +def is_ipv4_network(string_network): + if '/' in string_network: + try: + socket.inet_aton(string_network.split('/')[0]) + except OSError: + return False + else: + return False + return True + + def get_environ_proxies(url): """Return a dict of environment proxies.""" @@ -421,21 +455,12 @@ def get_environ_proxies(url): # the end of the netloc, both with and without the port. no_proxy = no_proxy.replace(' ', '').split(',') - ip = None - try: - ip = netaddr.IPAddress(netloc.split(':')[0]) - except netaddr.AddrFormatError: - pass - - if ip: + ip = netloc.split(':')[0] + if is_ipv4_address(ip): for proxy_ip in no_proxy: - proxy_ipaddress = None - try: - proxy_ipaddress = netaddr.IPNetwork(proxy_ip) - except (netaddr.AddrFormatError, ValueError): - continue - if proxy_ipaddress and ip in proxy_ipaddress: - return {} + if is_ipv4_network(proxy_ip): + if address_in_network(ip, proxy_ip): + return {} else: for host in no_proxy: if netloc.endswith(host) or netloc.split(':')[0].endswith(host): diff --git a/test_requests.py b/test_requests.py index cb91ade1..396970a4 100755 --- a/test_requests.py +++ b/test_requests.py @@ -913,11 +913,20 @@ class UtilsTestCase(unittest.TestCase): def test_get_environ_proxies_ip_ranges(self): """ Ensures that IP addresses are correctly matches with ranges in no_proxy variable """ from requests.utils import get_environ_proxies - os.environ['no_proxy'] = "127.0.0.1,localhost.localdomain,192.168.0.0/24,172.16.1.1" + os.environ['no_proxy'] = "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1" assert get_environ_proxies('http://192.168.0.1:5000/') == {} assert get_environ_proxies('http://192.168.0.1/') == {} assert get_environ_proxies('http://172.16.1.1/') == {} - assert get_environ_proxies('http://192.168.1.1:5000/') == {'no': os.environ['no_proxy']} + assert get_environ_proxies('http://172.16.1.1:5000/') == {} + assert get_environ_proxies('http://192.168.1.1:5000/') != {} + assert get_environ_proxies('http://192.168.1.1/') != {} + + def test_get_environ_proxies(self): + """ Ensures that IP addresses are correctly matches with ranges in no_proxy variable """ + from requests.utils import get_environ_proxies + os.environ['no_proxy'] = "127.0.0.1,localhost.localdomain,192.168.0.0/24,172.16.1.1" + assert get_environ_proxies('http://localhost.localdomain:5000/v1.0/') == {} + assert get_environ_proxies('http://www.requests.com/') != {} if __name__ == '__main__': unittest.main() From f0d3edf4090f3228f576cee665e3135052fbf303 Mon Sep 17 00:00:00 2001 From: Kamil Madac Date: Tue, 3 Dec 2013 16:30:47 +0100 Subject: [PATCH 07/19] Small test added --- test_requests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_requests.py b/test_requests.py index 396970a4..75858b5b 100755 --- a/test_requests.py +++ b/test_requests.py @@ -926,6 +926,7 @@ class UtilsTestCase(unittest.TestCase): from requests.utils import get_environ_proxies os.environ['no_proxy'] = "127.0.0.1,localhost.localdomain,192.168.0.0/24,172.16.1.1" assert get_environ_proxies('http://localhost.localdomain:5000/v1.0/') == {} + assert get_environ_proxies('http://www.requests.com:5000/v1.0/') != {} assert get_environ_proxies('http://www.requests.com/') != {} if __name__ == '__main__': From 61313903453ee68677c5afe0a8cb3e26f8215f9c Mon Sep 17 00:00:00 2001 From: Kamil Madac Date: Tue, 3 Dec 2013 16:41:07 +0100 Subject: [PATCH 08/19] Revert "Small test added" This reverts commit f0d3edf4090f3228f576cee665e3135052fbf303. --- test_requests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_requests.py b/test_requests.py index 75858b5b..396970a4 100755 --- a/test_requests.py +++ b/test_requests.py @@ -926,7 +926,6 @@ class UtilsTestCase(unittest.TestCase): from requests.utils import get_environ_proxies os.environ['no_proxy'] = "127.0.0.1,localhost.localdomain,192.168.0.0/24,172.16.1.1" assert get_environ_proxies('http://localhost.localdomain:5000/v1.0/') == {} - assert get_environ_proxies('http://www.requests.com:5000/v1.0/') != {} assert get_environ_proxies('http://www.requests.com/') != {} if __name__ == '__main__': From fe945345dd5d9b187632e9f9084cd101ee3a1211 Mon Sep 17 00:00:00 2001 From: Kamil Madac Date: Wed, 4 Dec 2013 09:31:56 +0100 Subject: [PATCH 09/19] Unnecessary files kmadac.xml deleted --- .idea/dictionaries/kmadac.xml | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .idea/dictionaries/kmadac.xml diff --git a/.idea/dictionaries/kmadac.xml b/.idea/dictionaries/kmadac.xml deleted file mode 100644 index 93bbf4db..00000000 --- a/.idea/dictionaries/kmadac.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - \ No newline at end of file From 0a4e65c0ed6bab6fbe9cfc4f4c4d349098b6f319 Mon Sep 17 00:00:00 2001 From: Kamil Madac Date: Wed, 4 Dec 2013 10:54:36 +0100 Subject: [PATCH 10/19] Tests added --- test_requests.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test_requests.py b/test_requests.py index 396970a4..8e6a1647 100755 --- a/test_requests.py +++ b/test_requests.py @@ -928,5 +928,16 @@ class UtilsTestCase(unittest.TestCase): assert get_environ_proxies('http://localhost.localdomain:5000/v1.0/') == {} assert get_environ_proxies('http://www.requests.com/') != {} + def test_is_ipv4_address(self): + from requests.utils import is_ipv4_address + assert is_ipv4_address('8.8.8.8') + assert not is_ipv4_address('8.8.8.8.8') + assert not is_ipv4_address('localhost.localdomain') + + def test_is_ipv4_network(self): + from requests.utils import is_ipv4_network + assert not is_ipv4_network('8.8.8.8') + assert is_ipv4_network('192.168.1.0/24') + if __name__ == '__main__': unittest.main() From 947248129ba3b0917c86c013f24d7499c15cb8ae Mon Sep 17 00:00:00 2001 From: Kamil Madac Date: Wed, 4 Dec 2013 11:47:40 +0100 Subject: [PATCH 11/19] Better comments --- requests/utils.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index d7852308..1bebbaaf 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -407,8 +407,12 @@ def requote_uri(uri): return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~") -def address_in_network(ip,net): - '''This function allows you to check if on IP belogs to a Network''' +def address_in_network(ip, net): + """ + This function allows you to check if on IP belongs to a Network subnet + Example: returns True if ip = 192.168.1.1 and net = 192.168.1.0/24 + returns False if ip = 192.168.1.1 and net = 192.168.100.0/24 + """ ipaddr = struct.unpack('=L', socket.inet_aton(ip))[0] netaddr, bits = net.split('/') netmask = struct.unpack('=L', socket.inet_aton(dotted_netmask(int(bits))))[0] @@ -417,6 +421,10 @@ def address_in_network(ip,net): def dotted_netmask(mask): + """ + Converts mask from /xx format to xxx.xxx.xxx.xxx + Example: if mask is /24 function returns 255.255.255.0 + """ bits = 0xffffffff ^ (1 << 32 - mask) - 1 return socket.inet_ntoa(struct.pack('>I', bits)) @@ -430,6 +438,7 @@ def is_ipv4_address(string_ip): def is_ipv4_network(string_network): + """Very simple check of the network format in no_proxy variable""" if '/' in string_network: try: socket.inet_aton(string_network.split('/')[0]) From 5b30e960eb6ebb62f22c52653f4b134e917594a2 Mon Sep 17 00:00:00 2001 From: Chase Sterling Date: Wed, 6 Nov 2013 19:39:56 -0500 Subject: [PATCH 12/19] Add a test case for request cookies persisting to session. refs #1728 --- test_requests.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test_requests.py b/test_requests.py index 1d68c81f..617e6a7f 100755 --- a/test_requests.py +++ b/test_requests.py @@ -165,7 +165,7 @@ class RequestsTestCase(unittest.TestCase): def test_cookie_persists_via_api(self): s = requests.session() - r = s.get(httpbin('redirect/1'), cookies={'foo':'bar'}) + r = s.get(httpbin('redirect/1'), cookies={'foo': 'bar'}) assert 'foo' in r.request.headers['Cookie'] assert 'foo' in r.history[0].request.headers['Cookie'] @@ -177,6 +177,12 @@ class RequestsTestCase(unittest.TestCase): # Session cookie should not be modified assert s.cookies['foo'] == 'bar' + def test_request_cookies_not_persisted(self): + s = requests.session() + s.get(httpbin('cookies'), cookies={'foo': 'baz'}) + # Sending a request with cookies should not add cookies to the session + assert not s.cookies + def test_generic_cookiejar_works(self): cj = cookielib.CookieJar() cookiejar_from_dict({'foo': 'bar'}, cj) From 8bfe0d873f382ec92e4000084bb7e00505bc29b2 Mon Sep 17 00:00:00 2001 From: Chase Sterling Date: Wed, 6 Nov 2013 21:52:21 -0500 Subject: [PATCH 13/19] Store the request cookiejar in PreparedRequest.cookies fix #1728 Conflicts: requests/sessions.py --- requests/auth.py | 4 +++- requests/models.py | 15 +++++++++------ requests/sessions.py | 6 ++---- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/requests/auth.py b/requests/auth.py index f87087df..ba979b59 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -16,6 +16,7 @@ import logging from base64 import b64encode from .compat import urlparse, str +from .cookies import extract_cookies_to_jar from .utils import parse_dict_header log = logging.getLogger(__name__) @@ -169,7 +170,8 @@ class HTTPDigestAuth(AuthBase): r.content r.raw.release_conn() prep = r.request.copy() - prep.prepare_cookies(r.cookies) + extract_cookies_to_jar(prep.cookies, r.request, r.raw) + prep.prepare_cookies(prep.cookies) prep.headers['Authorization'] = self.build_digest_header( prep.method, prep.url) diff --git a/requests/models.py b/requests/models.py index c68f9118..31d120e8 100644 --- a/requests/models.py +++ b/requests/models.py @@ -270,6 +270,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): self.url = None #: dictionary of HTTP headers. self.headers = None + #: The `CookieJar` used to create the Cookie header will be stored here + #: after prepare_cookies is called + self.cookies = None #: request body to send to the server. self.body = None #: dictionary of callback hooks, for internal usage. @@ -299,6 +302,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): p.method = self.method p.url = self.url p.headers = self.headers.copy() + p.cookies = self.cookies.copy() p.body = self.body p.hooks = self.hooks return p @@ -474,14 +478,13 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): """Prepares the given HTTP cookie data.""" if isinstance(cookies, cookielib.CookieJar): - cookies = cookies + self.cookies = cookies else: - cookies = cookiejar_from_dict(cookies) + self.cookies = cookiejar_from_dict(cookies) - if 'cookie' not in self.headers: - cookie_header = get_cookie_header(cookies, self) - if cookie_header is not None: - self.headers['Cookie'] = cookie_header + cookie_header = get_cookie_header(self.cookies, self) + if cookie_header is not None: + self.headers['Cookie'] = cookie_header def prepare_hooks(self, hooks): """Prepares the given hooks.""" diff --git a/requests/sessions.py b/requests/sessions.py index cdce6484..52adb7a0 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -153,7 +153,8 @@ class SessionRedirectMixin(object): except KeyError: pass - prepared_request.prepare_cookies(self.cookies) + extract_cookies_to_jar(prepared_request.cookies, prepared_request, resp.raw) + prepared_request.prepare_cookies(prepared_request.cookies) resp = self.send( prepared_request, @@ -345,9 +346,6 @@ class Session(SessionRedirectMixin): ) prep = self.prepare_request(req) - # Add param cookies to session cookies - self.cookies = merge_cookies(self.cookies, cookies) - proxies = proxies or {} # Gather clues from the surrounding environment. From e8bdcdb0a754ce3ec46fe167dfbd0e23f22e5ff7 Mon Sep 17 00:00:00 2001 From: Ian Cordasco Date: Wed, 4 Dec 2013 06:45:18 -0600 Subject: [PATCH 14/19] Address feedback from #1729 - Make the PreparedRequest's cookie jar an implementation detail --- requests/auth.py | 4 ++-- requests/models.py | 14 +++++++------- requests/sessions.py | 5 +++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/requests/auth.py b/requests/auth.py index ba979b59..6664cd80 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -170,8 +170,8 @@ class HTTPDigestAuth(AuthBase): r.content r.raw.release_conn() prep = r.request.copy() - extract_cookies_to_jar(prep.cookies, r.request, r.raw) - prep.prepare_cookies(prep.cookies) + extract_cookies_to_jar(prep._cookies, r.request, r.raw) + prep.prepare_cookies(prep._cookies) prep.headers['Authorization'] = self.build_digest_header( prep.method, prep.url) diff --git a/requests/models.py b/requests/models.py index 31d120e8..34dce181 100644 --- a/requests/models.py +++ b/requests/models.py @@ -270,9 +270,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): self.url = None #: dictionary of HTTP headers. self.headers = None - #: The `CookieJar` used to create the Cookie header will be stored here - #: after prepare_cookies is called - self.cookies = None + # The `CookieJar` used to create the Cookie header will be stored here + # after prepare_cookies is called + self._cookies = None #: request body to send to the server. self.body = None #: dictionary of callback hooks, for internal usage. @@ -302,7 +302,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): p.method = self.method p.url = self.url p.headers = self.headers.copy() - p.cookies = self.cookies.copy() + p._cookies = self._cookies.copy() p.body = self.body p.hooks = self.hooks return p @@ -478,11 +478,11 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): """Prepares the given HTTP cookie data.""" if isinstance(cookies, cookielib.CookieJar): - self.cookies = cookies + self._cookies = cookies else: - self.cookies = cookiejar_from_dict(cookies) + self._cookies = cookiejar_from_dict(cookies) - cookie_header = get_cookie_header(self.cookies, self) + cookie_header = get_cookie_header(self._cookies, self) if cookie_header is not None: self.headers['Cookie'] = cookie_header diff --git a/requests/sessions.py b/requests/sessions.py index 52adb7a0..06e17d4b 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -153,8 +153,9 @@ class SessionRedirectMixin(object): except KeyError: pass - extract_cookies_to_jar(prepared_request.cookies, prepared_request, resp.raw) - prepared_request.prepare_cookies(prepared_request.cookies) + extract_cookies_to_jar(prepared_request._cookies, + prepared_request, resp.raw) + prepared_request.prepare_cookies(prepared_request._cookies) resp = self.send( prepared_request, From 0d52ef7084a6839376be182dd91e9e003d3275ff Mon Sep 17 00:00:00 2001 From: Kamil Madac Date: Wed, 4 Dec 2013 15:10:24 +0100 Subject: [PATCH 15/19] Additional tests added --- test_requests.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test_requests.py b/test_requests.py index 8e6a1647..251d2ea0 100755 --- a/test_requests.py +++ b/test_requests.py @@ -939,5 +939,17 @@ class UtilsTestCase(unittest.TestCase): assert not is_ipv4_network('8.8.8.8') assert is_ipv4_network('192.168.1.0/24') + def test_dotted_netmask(self): + from requests.utils import dotted_netmask + assert dotted_netmask(8) == '255.0.0.0' + assert dotted_netmask(24) == '255.255.255.0' + assert dotted_netmask(25) == '255.255.255.128' + + def test_address_in_network(self): + from requests.utils import address_in_network + assert address_in_network('192.168.1.1', '192.168.1.0/24') + assert not address_in_network('172.16.0.1', '192.168.1.0/24') + + if __name__ == '__main__': unittest.main() From 59b29d02f8c4674a9922df53c9df9f960e6eaf34 Mon Sep 17 00:00:00 2001 From: Kamil Madac Date: Wed, 4 Dec 2013 15:11:10 +0100 Subject: [PATCH 16/19] Tiny comment fix --- requests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/utils.py b/requests/utils.py index 1bebbaaf..7b728c0d 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -423,7 +423,7 @@ def address_in_network(ip, net): def dotted_netmask(mask): """ Converts mask from /xx format to xxx.xxx.xxx.xxx - Example: if mask is /24 function returns 255.255.255.0 + Example: if mask is 24 function returns 255.255.255.0 """ bits = 0xffffffff ^ (1 << 32 - mask) - 1 return socket.inet_ntoa(struct.pack('>I', bits)) From 21c88c829aade9347d74b69546f2541ae2a66caf Mon Sep 17 00:00:00 2001 From: Kamil Madac Date: Thu, 5 Dec 2013 09:32:29 +0100 Subject: [PATCH 17/19] Broad exception fixed --- requests/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 7b728c0d..12ff74df 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -432,7 +432,7 @@ def dotted_netmask(mask): def is_ipv4_address(string_ip): try: socket.inet_aton(string_ip) - except BaseException: + except socket.error: return False return True @@ -442,7 +442,7 @@ def is_ipv4_network(string_network): if '/' in string_network: try: socket.inet_aton(string_network.split('/')[0]) - except OSError: + except socket.error: return False else: return False From f3bc8b6e1f470298467738f1a33c759eeb0474f4 Mon Sep 17 00:00:00 2001 From: Kamil Madac Date: Thu, 5 Dec 2013 09:36:49 +0100 Subject: [PATCH 18/19] Typo fixed --- requests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/utils.py b/requests/utils.py index 12ff74df..965b2550 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -409,7 +409,7 @@ def requote_uri(uri): def address_in_network(ip, net): """ - This function allows you to check if on IP belongs to a Network subnet + This function allows you to check if on IP belongs to a network subnet Example: returns True if ip = 192.168.1.1 and net = 192.168.1.0/24 returns False if ip = 192.168.1.1 and net = 192.168.100.0/24 """ From 1d42d9d2741d36c469fca38bc7e7a5383e3b7bd9 Mon Sep 17 00:00:00 2001 From: Kamil Madac Date: Thu, 5 Dec 2013 11:15:52 +0100 Subject: [PATCH 19/19] function is_ipv4_network renamed more detailed check of cidr format --- requests/utils.py | 16 ++++++++++++---- test_requests.py | 8 ++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index 965b2550..b272dc77 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -437,9 +437,17 @@ def is_ipv4_address(string_ip): return True -def is_ipv4_network(string_network): - """Very simple check of the network format in no_proxy variable""" - if '/' in string_network: +def is_valid_cidr(string_network): + """Very simple check of the cidr format in no_proxy variable""" + if string_network.count('/') == 1: + try: + mask = int(string_network.split('/')[1]) + except ValueError: + return False + + if mask < 1 or mask > 32: + return False + try: socket.inet_aton(string_network.split('/')[0]) except socket.error: @@ -467,7 +475,7 @@ def get_environ_proxies(url): ip = netloc.split(':')[0] if is_ipv4_address(ip): for proxy_ip in no_proxy: - if is_ipv4_network(proxy_ip): + if is_valid_cidr(proxy_ip): if address_in_network(ip, proxy_ip): return {} else: diff --git a/test_requests.py b/test_requests.py index 251d2ea0..c46030e4 100755 --- a/test_requests.py +++ b/test_requests.py @@ -934,10 +934,10 @@ class UtilsTestCase(unittest.TestCase): assert not is_ipv4_address('8.8.8.8.8') assert not is_ipv4_address('localhost.localdomain') - def test_is_ipv4_network(self): - from requests.utils import is_ipv4_network - assert not is_ipv4_network('8.8.8.8') - assert is_ipv4_network('192.168.1.0/24') + def test_is_valid_cidr(self): + from requests.utils import is_valid_cidr + assert not is_valid_cidr('8.8.8.8') + assert is_valid_cidr('192.168.1.0/24') def test_dotted_netmask(self): from requests.utils import dotted_netmask