diff --git a/AUTHORS.rst b/AUTHORS.rst index 2636fa6c..b0a7e241 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -143,3 +143,4 @@ Patches and Suggestions - Thomas Weißschuh @t-8ch - Jayson Vantuyl @kagato - Pengfei.X +- Kamil Madac diff --git a/requests/adapters.py b/requests/adapters.py index e065c759..b62f64c8 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -330,27 +330,40 @@ class HTTPAdapter(BaseAdapter): conn = conn.proxy_pool low_conn = conn._get_conn(timeout=timeout) - low_conn.putrequest(request.method, url, skip_accept_encoding=True) - for header, value in request.headers.items(): - low_conn.putheader(header, value) + try: + low_conn.putrequest(request.method, + url, + skip_accept_encoding=True) - low_conn.endheaders() + for header, value in request.headers.items(): + low_conn.putheader(header, value) - for i in request.body: - low_conn.send(hex(len(i))[2:].encode('utf-8')) - low_conn.send(b'\r\n') - low_conn.send(i) - low_conn.send(b'\r\n') - low_conn.send(b'0\r\n\r\n') + low_conn.endheaders() - r = low_conn.getresponse() - resp = HTTPResponse.from_httplib(r, - pool=conn, - connection=low_conn, - preload_content=False, - decode_content=False - ) + for i in request.body: + low_conn.send(hex(len(i))[2:].encode('utf-8')) + low_conn.send(b'\r\n') + low_conn.send(i) + low_conn.send(b'\r\n') + low_conn.send(b'0\r\n\r\n') + + r = low_conn.getresponse() + resp = HTTPResponse.from_httplib( + r, + pool=conn, + connection=low_conn, + preload_content=False, + decode_content=False + ) + except: + # If we hit any problems here, clean up the connection. + # Then, reraise so that we can handle the actual exception. + low_conn.close() + raise + else: + # All is well, return the connection to the pool. + conn._put_conn(low_conn) except socket.error as sockerr: raise ConnectionError(sockerr) diff --git a/requests/auth.py b/requests/auth.py index f87087df..6664cd80 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -16,6 +16,7 @@ import logging from base64 import b64encode from .compat import urlparse, str +from .cookies import extract_cookies_to_jar from .utils import parse_dict_header log = logging.getLogger(__name__) @@ -169,7 +170,8 @@ class HTTPDigestAuth(AuthBase): r.content r.raw.release_conn() prep = r.request.copy() - prep.prepare_cookies(r.cookies) + extract_cookies_to_jar(prep._cookies, r.request, r.raw) + prep.prepare_cookies(prep._cookies) prep.headers['Authorization'] = self.build_digest_header( prep.method, prep.url) diff --git a/requests/models.py b/requests/models.py index c68f9118..34dce181 100644 --- a/requests/models.py +++ b/requests/models.py @@ -270,6 +270,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): self.url = None #: dictionary of HTTP headers. self.headers = None + # The `CookieJar` used to create the Cookie header will be stored here + # after prepare_cookies is called + self._cookies = None #: request body to send to the server. self.body = None #: dictionary of callback hooks, for internal usage. @@ -299,6 +302,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): p.method = self.method p.url = self.url p.headers = self.headers.copy() + p._cookies = self._cookies.copy() p.body = self.body p.hooks = self.hooks return p @@ -474,14 +478,13 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): """Prepares the given HTTP cookie data.""" if isinstance(cookies, cookielib.CookieJar): - cookies = cookies + self._cookies = cookies else: - cookies = cookiejar_from_dict(cookies) + self._cookies = cookiejar_from_dict(cookies) - if 'cookie' not in self.headers: - cookie_header = get_cookie_header(cookies, self) - if cookie_header is not None: - self.headers['Cookie'] = cookie_header + cookie_header = get_cookie_header(self._cookies, self) + if cookie_header is not None: + self.headers['Cookie'] = cookie_header def prepare_hooks(self, hooks): """Prepares the given hooks.""" diff --git a/requests/sessions.py b/requests/sessions.py index cdce6484..06e17d4b 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -153,7 +153,9 @@ class SessionRedirectMixin(object): except KeyError: pass - prepared_request.prepare_cookies(self.cookies) + extract_cookies_to_jar(prepared_request._cookies, + prepared_request, resp.raw) + prepared_request.prepare_cookies(prepared_request._cookies) resp = self.send( prepared_request, @@ -345,9 +347,6 @@ class Session(SessionRedirectMixin): ) prep = self.prepare_request(req) - # Add param cookies to session cookies - self.cookies = merge_cookies(self.cookies, cookies) - proxies = proxies or {} # Gather clues from the surrounding environment. diff --git a/requests/utils.py b/requests/utils.py index eda43d33..c7e2b089 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -17,6 +17,8 @@ import os import platform import re import sys +import socket +import struct from . import __version__ from . import certs @@ -405,6 +407,56 @@ def requote_uri(uri): return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~") +def address_in_network(ip, net): + """ + This function allows you to check if on IP belongs to a network subnet + Example: returns True if ip = 192.168.1.1 and net = 192.168.1.0/24 + returns False if ip = 192.168.1.1 and net = 192.168.100.0/24 + """ + ipaddr = struct.unpack('=L', socket.inet_aton(ip))[0] + netaddr, bits = net.split('/') + netmask = struct.unpack('=L', socket.inet_aton(dotted_netmask(int(bits))))[0] + network = struct.unpack('=L', socket.inet_aton(netaddr))[0] & netmask + return (ipaddr & netmask) == (network & netmask) + + +def dotted_netmask(mask): + """ + Converts mask from /xx format to xxx.xxx.xxx.xxx + Example: if mask is 24 function returns 255.255.255.0 + """ + bits = 0xffffffff ^ (1 << 32 - mask) - 1 + return socket.inet_ntoa(struct.pack('>I', bits)) + + +def is_ipv4_address(string_ip): + try: + socket.inet_aton(string_ip) + except socket.error: + return False + return True + + +def is_valid_cidr(string_network): + """Very simple check of the cidr format in no_proxy variable""" + if string_network.count('/') == 1: + try: + mask = int(string_network.split('/')[1]) + except ValueError: + return False + + if mask < 1 or mask > 32: + return False + + try: + socket.inet_aton(string_network.split('/')[0]) + except socket.error: + return False + else: + return False + return True + + def get_environ_proxies(url): """Return a dict of environment proxies.""" @@ -420,11 +472,18 @@ def get_environ_proxies(url): # the end of the netloc, both with and without the port. no_proxy = no_proxy.replace(' ', '').split(',') - for host in no_proxy: - if netloc.endswith(host) or netloc.split(':')[0].endswith(host): - # The URL does match something in no_proxy, so we don't want - # to apply the proxies on this URL. - return {} + ip = netloc.split(':')[0] + if is_ipv4_address(ip): + for proxy_ip in no_proxy: + if is_valid_cidr(proxy_ip): + if address_in_network(ip, proxy_ip): + return {} + else: + for host in no_proxy: + if netloc.endswith(host) or netloc.split(':')[0].endswith(host): + # The URL does match something in no_proxy, so we don't want + # to apply the proxies on this URL. + return {} # If the system proxy settings indicate that this URL should be bypassed, # don't proxy. diff --git a/test_requests.py b/test_requests.py index 1d68c81f..a800ba71 100755 --- a/test_requests.py +++ b/test_requests.py @@ -165,7 +165,7 @@ class RequestsTestCase(unittest.TestCase): def test_cookie_persists_via_api(self): s = requests.session() - r = s.get(httpbin('redirect/1'), cookies={'foo':'bar'}) + r = s.get(httpbin('redirect/1'), cookies={'foo': 'bar'}) assert 'foo' in r.request.headers['Cookie'] assert 'foo' in r.history[0].request.headers['Cookie'] @@ -177,6 +177,12 @@ class RequestsTestCase(unittest.TestCase): # Session cookie should not be modified assert s.cookies['foo'] == 'bar' + def test_request_cookies_not_persisted(self): + s = requests.session() + s.get(httpbin('cookies'), cookies={'foo': 'baz'}) + # Sending a request with cookies should not add cookies to the session + assert not s.cookies + def test_generic_cookiejar_works(self): cj = cookielib.CookieJar() cookiejar_from_dict({'foo': 'bar'}, cj) @@ -957,5 +963,46 @@ class UtilsTestCase(unittest.TestCase): else: assert super_len(cStringIO.StringIO('but some how, some way...')) == 25 + def test_get_environ_proxies_ip_ranges(self): + """ Ensures that IP addresses are correctly matches with ranges in no_proxy variable """ + from requests.utils import get_environ_proxies + os.environ['no_proxy'] = "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1" + assert get_environ_proxies('http://192.168.0.1:5000/') == {} + assert get_environ_proxies('http://192.168.0.1/') == {} + assert get_environ_proxies('http://172.16.1.1/') == {} + assert get_environ_proxies('http://172.16.1.1:5000/') == {} + assert get_environ_proxies('http://192.168.1.1:5000/') != {} + assert get_environ_proxies('http://192.168.1.1/') != {} + + def test_get_environ_proxies(self): + """ Ensures that IP addresses are correctly matches with ranges in no_proxy variable """ + from requests.utils import get_environ_proxies + os.environ['no_proxy'] = "127.0.0.1,localhost.localdomain,192.168.0.0/24,172.16.1.1" + assert get_environ_proxies('http://localhost.localdomain:5000/v1.0/') == {} + assert get_environ_proxies('http://www.requests.com/') != {} + + def test_is_ipv4_address(self): + from requests.utils import is_ipv4_address + assert is_ipv4_address('8.8.8.8') + assert not is_ipv4_address('8.8.8.8.8') + assert not is_ipv4_address('localhost.localdomain') + + def test_is_valid_cidr(self): + from requests.utils import is_valid_cidr + assert not is_valid_cidr('8.8.8.8') + assert is_valid_cidr('192.168.1.0/24') + + def test_dotted_netmask(self): + from requests.utils import dotted_netmask + assert dotted_netmask(8) == '255.0.0.0' + assert dotted_netmask(24) == '255.255.255.0' + assert dotted_netmask(25) == '255.255.255.128' + + def test_address_in_network(self): + from requests.utils import address_in_network + assert address_in_network('192.168.1.1', '192.168.1.0/24') + assert not address_in_network('172.16.0.1', '192.168.1.0/24') + + if __name__ == '__main__': unittest.main()