Merge remote-tracking branch 'origin/master'

AUTHORS.rst

@@ -143,3 +143,4 @@ Patches and Suggestions
 - Thomas Weißschuh <thomas@t-8ch.de> @t-8ch
 - Jayson Vantuyl <jayson@aggressive.ly> @kagato
 - Pengfei.X <pengphy@gmail.com>
+- Kamil Madac <kamil.madac@gmail.com>

requests/adapters.py

@@ -330,27 +330,40 @@ class HTTPAdapter(BaseAdapter):
                     conn = conn.proxy_pool
 
                 low_conn = conn._get_conn(timeout=timeout)
-                low_conn.putrequest(request.method, url, skip_accept_encoding=True)
 
-                for header, value in request.headers.items():
-                    low_conn.putheader(header, value)
+                try:
+                    low_conn.putrequest(request.method,
+                                        url,
+                                        skip_accept_encoding=True)
 
-                low_conn.endheaders()
+                    for header, value in request.headers.items():
+                        low_conn.putheader(header, value)
 
-                for i in request.body:
-                    low_conn.send(hex(len(i))[2:].encode('utf-8'))
-                    low_conn.send(b'\r\n')
-                    low_conn.send(i)
-                    low_conn.send(b'\r\n')
-                low_conn.send(b'0\r\n\r\n')
+                    low_conn.endheaders()
 
-                r = low_conn.getresponse()
-                resp = HTTPResponse.from_httplib(r,
-                    pool=conn,
-                    connection=low_conn,
-                    preload_content=False,
-                    decode_content=False
-                )
+                    for i in request.body:
+                        low_conn.send(hex(len(i))[2:].encode('utf-8'))
+                        low_conn.send(b'\r\n')
+                        low_conn.send(i)
+                        low_conn.send(b'\r\n')
+                    low_conn.send(b'0\r\n\r\n')
+
+                    r = low_conn.getresponse()
+                    resp = HTTPResponse.from_httplib(
+                        r,
+                        pool=conn,
+                        connection=low_conn,
+                        preload_content=False,
+                        decode_content=False
+                    )
+                except:
+                    # If we hit any problems here, clean up the connection.
+                    # Then, reraise so that we can handle the actual exception.
+                    low_conn.close()
+                    raise
+                else:
+                    # All is well, return the connection to the pool.
+                    conn._put_conn(low_conn)
 
         except socket.error as sockerr:
             raise ConnectionError(sockerr)

requests/auth.py

@@ -16,6 +16,7 @@ import logging
 from base64 import b64encode
 
 from .compat import urlparse, str
+from .cookies import extract_cookies_to_jar
 from .utils import parse_dict_header
 
 log = logging.getLogger(__name__)
@@ -169,7 +170,8 @@ class HTTPDigestAuth(AuthBase):
             r.content
             r.raw.release_conn()
             prep = r.request.copy()
-            prep.prepare_cookies(r.cookies)
+            extract_cookies_to_jar(prep._cookies, r.request, r.raw)
+            prep.prepare_cookies(prep._cookies)
 
             prep.headers['Authorization'] = self.build_digest_header(
                 prep.method, prep.url)

requests/models.py

@@ -270,6 +270,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         self.url = None
         #: dictionary of HTTP headers.
         self.headers = None
+        # The `CookieJar` used to create the Cookie header will be stored here
+        # after prepare_cookies is called
+        self._cookies = None
         #: request body to send to the server.
         self.body = None
         #: dictionary of callback hooks, for internal usage.
@@ -299,6 +302,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         p.method = self.method
         p.url = self.url
         p.headers = self.headers.copy()
+        p._cookies = self._cookies.copy()
         p.body = self.body
         p.hooks = self.hooks
         return p
@@ -474,14 +478,13 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
         """Prepares the given HTTP cookie data."""
 
         if isinstance(cookies, cookielib.CookieJar):
-            cookies = cookies
+            self._cookies = cookies
         else:
-            cookies = cookiejar_from_dict(cookies)
+            self._cookies = cookiejar_from_dict(cookies)
 
-        if 'cookie' not in self.headers:
-            cookie_header = get_cookie_header(cookies, self)
-            if cookie_header is not None:
-                self.headers['Cookie'] = cookie_header
+        cookie_header = get_cookie_header(self._cookies, self)
+        if cookie_header is not None:
+            self.headers['Cookie'] = cookie_header
 
     def prepare_hooks(self, hooks):
         """Prepares the given hooks."""

requests/sessions.py

@@ -153,7 +153,9 @@ class SessionRedirectMixin(object):
             except KeyError:
                 pass
 
-            prepared_request.prepare_cookies(self.cookies)
+            extract_cookies_to_jar(prepared_request._cookies,
+                                   prepared_request, resp.raw)
+            prepared_request.prepare_cookies(prepared_request._cookies)
 
             resp = self.send(
                 prepared_request,
@@ -345,9 +347,6 @@ class Session(SessionRedirectMixin):
         )
         prep = self.prepare_request(req)
 
-        # Add param cookies to session cookies
-        self.cookies = merge_cookies(self.cookies, cookies)
-
         proxies = proxies or {}
 
         # Gather clues from the surrounding environment.

requests/utils.py

@@ -17,6 +17,8 @@ import os
 import platform
 import re
 import sys
+import socket
+import struct
 
 from . import __version__
 from . import certs
@@ -405,6 +407,56 @@ def requote_uri(uri):
     return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~")
 
 
+def address_in_network(ip, net):
+    """
+    This function allows you to check if on IP belongs to a network subnet
+    Example: returns True if ip = 192.168.1.1 and net = 192.168.1.0/24
+             returns False if ip = 192.168.1.1 and net = 192.168.100.0/24
+    """
+    ipaddr = struct.unpack('=L', socket.inet_aton(ip))[0]
+    netaddr, bits = net.split('/')
+    netmask = struct.unpack('=L', socket.inet_aton(dotted_netmask(int(bits))))[0]
+    network = struct.unpack('=L', socket.inet_aton(netaddr))[0] & netmask
+    return (ipaddr & netmask) == (network & netmask)
+
+
+def dotted_netmask(mask):
+    """
+    Converts mask from /xx format to xxx.xxx.xxx.xxx
+    Example: if mask is 24 function returns 255.255.255.0
+    """
+    bits = 0xffffffff ^ (1 << 32 - mask) - 1
+    return socket.inet_ntoa(struct.pack('>I', bits))
+
+
+def is_ipv4_address(string_ip):
+    try:
+        socket.inet_aton(string_ip)
+    except socket.error:
+        return False
+    return True
+
+
+def is_valid_cidr(string_network):
+    """Very simple check of the cidr format in no_proxy variable"""
+    if string_network.count('/') == 1:
+        try:
+            mask = int(string_network.split('/')[1])
+        except ValueError:
+            return False
+
+        if mask < 1 or mask > 32:
+            return False
+
+        try:
+            socket.inet_aton(string_network.split('/')[0])
+        except socket.error:
+            return False
+    else:
+        return False
+    return True
+
+
 def get_environ_proxies(url):
     """Return a dict of environment proxies."""
 
@@ -420,11 +472,18 @@ def get_environ_proxies(url):
         # the end of the netloc, both with and without the port.
         no_proxy = no_proxy.replace(' ', '').split(',')
 
-        for host in no_proxy:
-            if netloc.endswith(host) or netloc.split(':')[0].endswith(host):
-                # The URL does match something in no_proxy, so we don't want
-                # to apply the proxies on this URL.
-                return {}
+        ip = netloc.split(':')[0]
+        if is_ipv4_address(ip):
+            for proxy_ip in no_proxy:
+                if is_valid_cidr(proxy_ip):
+                    if address_in_network(ip, proxy_ip):
+                        return {}
+        else:
+            for host in no_proxy:
+                if netloc.endswith(host) or netloc.split(':')[0].endswith(host):
+                    # The URL does match something in no_proxy, so we don't want
+                    # to apply the proxies on this URL.
+                    return {}
 
     # If the system proxy settings indicate that this URL should be bypassed,
     # don't proxy.

test_requests.py

@@ -165,7 +165,7 @@ class RequestsTestCase(unittest.TestCase):
 
     def test_cookie_persists_via_api(self):
         s = requests.session()
-        r = s.get(httpbin('redirect/1'), cookies={'foo':'bar'})
+        r = s.get(httpbin('redirect/1'), cookies={'foo': 'bar'})
         assert 'foo' in r.request.headers['Cookie']
         assert 'foo' in r.history[0].request.headers['Cookie']
 
@@ -177,6 +177,12 @@ class RequestsTestCase(unittest.TestCase):
         # Session cookie should not be modified
         assert s.cookies['foo'] == 'bar'
 
+    def test_request_cookies_not_persisted(self):
+        s = requests.session()
+        s.get(httpbin('cookies'), cookies={'foo': 'baz'})
+        # Sending a request with cookies should not add cookies to the session
+        assert not s.cookies
+
     def test_generic_cookiejar_works(self):
         cj = cookielib.CookieJar()
         cookiejar_from_dict({'foo': 'bar'}, cj)
@@ -957,5 +963,46 @@ class UtilsTestCase(unittest.TestCase):
         else:
             assert super_len(cStringIO.StringIO('but some how, some way...')) == 25
 
+    def test_get_environ_proxies_ip_ranges(self):
+        """ Ensures that IP addresses are correctly matches with ranges in no_proxy variable """
+        from requests.utils import get_environ_proxies
+        os.environ['no_proxy'] = "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1"
+        assert get_environ_proxies('http://192.168.0.1:5000/') == {}
+        assert get_environ_proxies('http://192.168.0.1/') == {}
+        assert get_environ_proxies('http://172.16.1.1/') == {}
+        assert get_environ_proxies('http://172.16.1.1:5000/') == {}
+        assert get_environ_proxies('http://192.168.1.1:5000/') != {}
+        assert get_environ_proxies('http://192.168.1.1/') != {}
+
+    def test_get_environ_proxies(self):
+        """ Ensures that IP addresses are correctly matches with ranges in no_proxy variable """
+        from requests.utils import get_environ_proxies
+        os.environ['no_proxy'] = "127.0.0.1,localhost.localdomain,192.168.0.0/24,172.16.1.1"
+        assert get_environ_proxies('http://localhost.localdomain:5000/v1.0/') == {}
+        assert get_environ_proxies('http://www.requests.com/') != {}
+
+    def test_is_ipv4_address(self):
+        from requests.utils import is_ipv4_address
+        assert is_ipv4_address('8.8.8.8')
+        assert not is_ipv4_address('8.8.8.8.8')
+        assert not is_ipv4_address('localhost.localdomain')
+
+    def test_is_valid_cidr(self):
+        from requests.utils import is_valid_cidr
+        assert not is_valid_cidr('8.8.8.8')
+        assert is_valid_cidr('192.168.1.0/24')
+
+    def test_dotted_netmask(self):
+        from requests.utils import dotted_netmask
+        assert dotted_netmask(8) == '255.0.0.0'
+        assert dotted_netmask(24) == '255.255.255.0'
+        assert dotted_netmask(25) == '255.255.255.128'
+
+    def test_address_in_network(self):
+        from requests.utils import address_in_network
+        assert address_in_network('192.168.1.1', '192.168.1.0/24')
+        assert not address_in_network('172.16.0.1', '192.168.1.0/24')
+
+
 if __name__ == '__main__':
     unittest.main()