From 99825de025fe47372ce8fc09f1461a7f0576f4f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Tue, 24 Nov 2015 23:08:38 +0100 Subject: [PATCH 001/182] Implemented dummy socket server interface and added one test --- dummyserver/__init__.py | 0 dummyserver/server.py | 38 ++++++++++++++++++++++++++++++++++++++ test_requests.py | 19 +++++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 dummyserver/__init__.py create mode 100644 dummyserver/server.py diff --git a/dummyserver/__init__.py b/dummyserver/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dummyserver/server.py b/dummyserver/server.py new file mode 100644 index 00000000..e8a90bc4 --- /dev/null +++ b/dummyserver/server.py @@ -0,0 +1,38 @@ +#!/usr/bin/python + +import threading, socket + + + +""" +Dummy server using for unit testing +""" + +class Server(threading.Thread): + def __init__(self, handler, host='localhost', port=8021): + threading.Thread.__init__(self) + self.handler = handler + self.host = host + self.port = port + self.ready_event = threading.Event() + self.stop_event = threading.Event() + + def run(self): + sock = socket.socket() + sock.bind((self.host, self.port)) + sock.listen(0) + self.ready_event.set() + self.handler(sock) + self.stop_event.set() + sock.close() + + def __enter__(self): + self.start() + self.ready_event.wait() + return self.host, self.port + + def __exit__(self, exc_type, exc_value, traceback): + if exc_type is None: + self.stop_event.wait() + return False # allow exceptions to propagate + diff --git a/test_requests.py b/test_requests.py index e83cbc44..5bf1e402 100755 --- a/test_requests.py +++ b/test_requests.py @@ -29,6 +29,8 @@ from requests.structures import CaseInsensitiveDict from requests.sessions import SessionRedirectMixin from requests.models import urlencode from requests.hooks import default_hooks +from dummyserver.server import Server +import socket try: import StringIO @@ -1736,6 +1738,23 @@ def test_vendor_aliases(): with pytest.raises(ImportError): from requests.packages import webbrowser +class TestDummyServer(unittest.TestCase): + def test_basic(self): + question = "sucess?" + answer = "yeah, success" + def handler(server_sock): + sock, _ = server_sock.accept() + text = sock.recv(1000) + assert text == question + sock.send(answer) + + with Server(handler) as (host, port): + sock = socket.socket() + sock.connect((host, port)) + sock.send(question) + text = sock.recv(1000) + assert text == answer + sock.close() if __name__ == '__main__': unittest.main() From 9a2cc99b068b2aaa572f52b4516852b239577c34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Wed, 25 Nov 2015 13:52:53 +0100 Subject: [PATCH 002/182] Put docstring inside Server class --- dummyserver/server.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dummyserver/server.py b/dummyserver/server.py index e8a90bc4..22c5d90f 100644 --- a/dummyserver/server.py +++ b/dummyserver/server.py @@ -4,11 +4,10 @@ import threading, socket -""" -Dummy server using for unit testing -""" class Server(threading.Thread): + """ Dummy server using for unit testing """ + def __init__(self, handler, host='localhost', port=8021): threading.Thread.__init__(self) self.handler = handler From b806ce15ba5d726f94f58aa247fe40b6e2972fb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Wed, 25 Nov 2015 13:53:54 +0100 Subject: [PATCH 003/182] Make strings in DummyServer test bytestrings --- test_requests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_requests.py b/test_requests.py index 5bf1e402..89d9c75b 100755 --- a/test_requests.py +++ b/test_requests.py @@ -1740,8 +1740,8 @@ def test_vendor_aliases(): class TestDummyServer(unittest.TestCase): def test_basic(self): - question = "sucess?" - answer = "yeah, success" + question = b"sucess?" + answer = b"yeah, success" def handler(server_sock): sock, _ = server_sock.accept() text = sock.recv(1000) From 47e96c4e57964dd2db4806b2c24e02a89f3ae8bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Wed, 25 Nov 2015 14:05:21 +0100 Subject: [PATCH 004/182] Renamed dummyserver to testserver --- test_requests.py | 4 ++-- testserver/.server.py.swo | Bin 0 -> 12288 bytes {dummyserver => testserver}/__init__.py | 0 {dummyserver => testserver}/server.py | 5 +---- 4 files changed, 3 insertions(+), 6 deletions(-) create mode 100644 testserver/.server.py.swo rename {dummyserver => testserver}/__init__.py (100%) rename {dummyserver => testserver}/server.py (94%) diff --git a/test_requests.py b/test_requests.py index 89d9c75b..926abe2e 100755 --- a/test_requests.py +++ b/test_requests.py @@ -29,7 +29,7 @@ from requests.structures import CaseInsensitiveDict from requests.sessions import SessionRedirectMixin from requests.models import urlencode from requests.hooks import default_hooks -from dummyserver.server import Server +from testserver.server import Server import socket try: @@ -1738,7 +1738,7 @@ def test_vendor_aliases(): with pytest.raises(ImportError): from requests.packages import webbrowser -class TestDummyServer(unittest.TestCase): +class TestTestServer(unittest.TestCase): def test_basic(self): question = b"sucess?" answer = b"yeah, success" diff --git a/testserver/.server.py.swo b/testserver/.server.py.swo new file mode 100644 index 0000000000000000000000000000000000000000..eb2d89d951ba5e633d389c0c2bc56934ae95a5e2 GIT binary patch literal 12288 zcmeI2y=xRf9EK-?VAQB6Ho;SH{&_+RQ6tohwQG91V?k=a8A_xY41JBK7=j)wkZUZ4(DbG$_ z;b)A~4CM%8jYl)HD~0<#Uq>0MgjVD_zHa$zxp@BaU)!_Bo9Tq5nq@D@QYe;$SJzb} zq>?42KSfvS(a zVL2*aV7f^cXHHLJs%9X#Bl%U|6Jty_RK*-OO|k4K)6DBhimi^xRo!wUQRFJLY*E2W zkcDJiJ9L^k4p8%@DkfP1KUDDwSR`%>g;j0-F^a>poryW)7&d@RlA+IV4W zPd*-p7Ru0Tx02~?hLIP~uw{c=&l00mP1|$4dE3?OfAwLc=s2uk}?*BqWE~al4pJgA1%RKIHngn|NFR literal 0 HcmV?d00001 diff --git a/dummyserver/__init__.py b/testserver/__init__.py similarity index 100% rename from dummyserver/__init__.py rename to testserver/__init__.py diff --git a/dummyserver/server.py b/testserver/server.py similarity index 94% rename from dummyserver/server.py rename to testserver/server.py index 22c5d90f..14349fb9 100644 --- a/dummyserver/server.py +++ b/testserver/server.py @@ -2,11 +2,8 @@ import threading, socket - - - class Server(threading.Thread): - """ Dummy server using for unit testing """ + """ Basic socket server used for unit testing """ def __init__(self, handler, host='localhost', port=8021): threading.Thread.__init__(self) From ab0f063d85d01a707516c84881c0a698edc526cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Wed, 25 Nov 2015 14:15:51 +0100 Subject: [PATCH 005/182] Made TestServer.Server listen on an arbitrary open port by default --- testserver/.server.py.swo | Bin 12288 -> 0 bytes testserver/server.py | 7 +++++-- 2 files changed, 5 insertions(+), 2 deletions(-) delete mode 100644 testserver/.server.py.swo diff --git a/testserver/.server.py.swo b/testserver/.server.py.swo deleted file mode 100644 index eb2d89d951ba5e633d389c0c2bc56934ae95a5e2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2y=xRf9EK-?VAQB6Ho;SH{&_+RQ6tohwQG91V?k=a8A_xY41JBK7=j)wkZUZ4(DbG$_ z;b)A~4CM%8jYl)HD~0<#Uq>0MgjVD_zHa$zxp@BaU)!_Bo9Tq5nq@D@QYe;$SJzb} zq>?42KSfvS(a zVL2*aV7f^cXHHLJs%9X#Bl%U|6Jty_RK*-OO|k4K)6DBhimi^xRo!wUQRFJLY*E2W zkcDJiJ9L^k4p8%@DkfP1KUDDwSR`%>g;j0-F^a>poryW)7&d@RlA+IV4W zPd*-p7Ru0Tx02~?hLIP~uw{c=&l00mP1|$4dE3?OfAwLc=s2uk}?*BqWE~al4pJgA1%RKIHngn|NFR diff --git a/testserver/server.py b/testserver/server.py index 14349fb9..689aaaf8 100644 --- a/testserver/server.py +++ b/testserver/server.py @@ -3,9 +3,9 @@ import threading, socket class Server(threading.Thread): - """ Basic socket server used for unit testing """ + """ Dummy server using for unit testing """ - def __init__(self, handler, host='localhost', port=8021): + def __init__(self, handler, host='localhost', port=0): threading.Thread.__init__(self) self.handler = handler self.host = host @@ -16,6 +16,9 @@ class Server(threading.Thread): def run(self): sock = socket.socket() sock.bind((self.host, self.port)) + + # update port in case self.port = 0 + self.port = sock.getsockname()[1] sock.listen(0) self.ready_event.set() self.handler(sock) From 2ad3f079821769062ad6b8e91c927b1b345895d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Wed, 25 Nov 2015 14:41:03 +0100 Subject: [PATCH 006/182] Make exceptions in the testserver.Server thread not block the main thread --- testserver/server.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/testserver/server.py b/testserver/server.py index 689aaaf8..6fa6464e 100644 --- a/testserver/server.py +++ b/testserver/server.py @@ -14,16 +14,23 @@ class Server(threading.Thread): self.stop_event = threading.Event() def run(self): + try: + sock = self._create_socket_and_bind() + # in case self.port = 0 + self.port = sock.getsockname()[1] + self.ready_event.set() + self.handler(sock) + + finally: + self.ready_event.set() # just in case of exception + self.stop_event.set() + sock.close() + + def _create_socket_and_bind(self): sock = socket.socket() sock.bind((self.host, self.port)) - - # update port in case self.port = 0 - self.port = sock.getsockname()[1] sock.listen(0) - self.ready_event.set() - self.handler(sock) - self.stop_event.set() - sock.close() + return sock def __enter__(self): self.start() From 11ad502b80a10d3c4fe5631509f893a7bd20c2e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Thu, 26 Nov 2015 00:29:53 +0100 Subject: [PATCH 007/182] Added basic_response_server classmethod and tested it --- test_requests.py | 7 +++++++ testserver/server.py | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/test_requests.py b/test_requests.py index 926abe2e..75253eb3 100755 --- a/test_requests.py +++ b/test_requests.py @@ -1756,5 +1756,12 @@ class TestTestServer(unittest.TestCase): assert text == answer sock.close() + def test_basic_response(self): + with Server.basic_response_server() as (host, port): + r = requests.get('http://{}:{}'.format(host, port)) + assert r.status_code == 200 + assert r.text == '' + assert r.headers['Content-Length'] == '0' + if __name__ == '__main__': unittest.main() diff --git a/testserver/server.py b/testserver/server.py index 6fa6464e..f19cc3c7 100644 --- a/testserver/server.py +++ b/testserver/server.py @@ -13,6 +13,19 @@ class Server(threading.Thread): self.ready_event = threading.Event() self.stop_event = threading.Event() + @classmethod + def basic_response_server(cls, host='localhost', port=0): + def basic_response_handler(server_sock): + sock, _ = server_sock.accept() + sock.send( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 0\r\n' + b'\r\n' + ) + + server = Server(basic_response_handler, host=host, port=port) + return server + def run(self): try: sock = self._create_socket_and_bind() From f7917c305a4f05a86a3ae058036e2205dc0f47e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Thu, 26 Nov 2015 01:04:27 +0100 Subject: [PATCH 008/182] Make server set stop_event right after closing its socket --- testserver/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testserver/server.py b/testserver/server.py index f19cc3c7..e329a9fc 100644 --- a/testserver/server.py +++ b/testserver/server.py @@ -36,8 +36,8 @@ class Server(threading.Thread): finally: self.ready_event.set() # just in case of exception - self.stop_event.set() sock.close() + self.stop_event.set() def _create_socket_and_bind(self): sock = socket.socket() From b4b4661f461a132c6815daa3719a3200553464a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Thu, 26 Nov 2015 01:05:16 +0100 Subject: [PATCH 009/182] Implemented test to verify that the context manager exits after the server is closed --- test_requests.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test_requests.py b/test_requests.py index 75253eb3..2d433f95 100755 --- a/test_requests.py +++ b/test_requests.py @@ -1756,6 +1756,16 @@ class TestTestServer(unittest.TestCase): assert text == answer sock.close() + def test_server_closes(self): + with Server.basic_response_server() as (host, port): + sock = socket.socket() + sock.connect((host, port)) + + with pytest.raises(socket.error): + new_sock = socket.socket() + new_sock.connect((host, port)) + sock.close() + def test_basic_response(self): with Server.basic_response_server() as (host, port): r = requests.get('http://{}:{}'.format(host, port)) From 6adabacd81d8e453c3915f8a98692c9891df022b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Thu, 26 Nov 2015 01:11:18 +0100 Subject: [PATCH 010/182] Close socket properly in test_server_closes --- test_requests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requests.py b/test_requests.py index 2d433f95..e54e6f01 100755 --- a/test_requests.py +++ b/test_requests.py @@ -1760,11 +1760,11 @@ class TestTestServer(unittest.TestCase): with Server.basic_response_server() as (host, port): sock = socket.socket() sock.connect((host, port)) + sock.close() with pytest.raises(socket.error): new_sock = socket.socket() new_sock.connect((host, port)) - sock.close() def test_basic_response(self): with Server.basic_response_server() as (host, port): From 6551c290fdbf0b29e4534abef1b04f7485a88aad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Wed, 2 Dec 2015 01:52:32 +0100 Subject: [PATCH 011/182] Allow main thread to block testserver before closing --- testserver/server.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/testserver/server.py b/testserver/server.py index e329a9fc..5c5d8069 100644 --- a/testserver/server.py +++ b/testserver/server.py @@ -1,39 +1,47 @@ #!/usr/bin/python -import threading, socket +import threading +import socket + +def consume_socket(sock, chunks=65536): + while not sock.recv(chunks).endswith(b'\r\n\r\n'): + pass + class Server(threading.Thread): """ Dummy server using for unit testing """ - def __init__(self, handler, host='localhost', port=0): + def __init__(self, handler, host='localhost', port=0, requests_to_handle=1, wait_to_close_event=None): threading.Thread.__init__(self) self.handler = handler self.host = host self.port = port + self.requests_to_handle = requests_to_handle + + self.wait_to_close_event = wait_to_close_event self.ready_event = threading.Event() self.stop_event = threading.Event() @classmethod - def basic_response_server(cls, host='localhost', port=0): - def basic_response_handler(server_sock): - sock, _ = server_sock.accept() + def basic_response_server(cls, **kwargs): + def basic_response_handler(sock): sock.send( b'HTTP/1.1 200 OK\r\n' b'Content-Length: 0\r\n' b'\r\n' ) - server = Server(basic_response_handler, host=host, port=port) + server = Server(basic_response_handler, **kwargs) return server + def run(self): try: sock = self._create_socket_and_bind() # in case self.port = 0 self.port = sock.getsockname()[1] self.ready_event.set() - self.handler(sock) - + self._handle_requests_and_close_server(sock) finally: self.ready_event.set() # just in case of exception sock.close() @@ -45,6 +53,14 @@ class Server(threading.Thread): sock.listen(0) return sock + def _handle_requests_and_close_server(self, server_sock): + for _ in range(self.requests_to_handle): + sock = server_sock.accept()[0] + self.handler(sock) + + if self.wait_to_close_event: + self.wait_to_close_event.wait() + def __enter__(self): self.start() self.ready_event.wait() From aac32ac56e60ac220a15c33d207306806c193d1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Wed, 2 Dec 2015 01:53:39 +0100 Subject: [PATCH 012/182] Updated tests relying on the old testserver and added basic test for chunked uploads --- test_requests.py | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/test_requests.py b/test_requests.py index e54e6f01..5c91c20a 100755 --- a/test_requests.py +++ b/test_requests.py @@ -31,6 +31,8 @@ from requests.models import urlencode from requests.hooks import default_hooks from testserver.server import Server import socket +import threading +import time try: import StringIO @@ -1179,6 +1181,24 @@ class TestContentEncodingDetection(unittest.TestCase): encodings = requests.utils.get_encodings_from_content(content) assert encodings == ['HTML5', 'HTML4', 'XML'] + def test_chunked_upload(self): + """ can safely send generators """ + block_server = threading.Event() + server = Server.basic_response_server(wait_to_close_event=block_server) + data = (i for i in [b'a', b'b', b'c']) + + with server as (host, port): + url = 'http://{}:{}/'.format(host, port) + r = requests.post(url, data=data, stream=True) + block_server.set() # release server block + + assert r.status_code == 200 + assert r.request.headers['Transfer-Encoding'] == 'chunked' + + + + + class TestCaseInsensitiveDict(unittest.TestCase): @@ -1742,8 +1762,7 @@ class TestTestServer(unittest.TestCase): def test_basic(self): question = b"sucess?" answer = b"yeah, success" - def handler(server_sock): - sock, _ = server_sock.accept() + def handler(sock): text = sock.recv(1000) assert text == question sock.send(answer) @@ -1773,5 +1792,16 @@ class TestTestServer(unittest.TestCase): assert r.text == '' assert r.headers['Content-Length'] == '0' + def test_basic_waiting_server(self): + block_server = threading.Event() + + with Server.basic_response_server(wait_to_close_event=block_server) as (host, port): + sock = socket.socket() + sock.connect((host, port)) + sock.send(b'send something') + time.sleep(2.5) + sock.send(b'still alive') + block_server.set() # release server block + if __name__ == '__main__': unittest.main() From 5ecf789a6f2c50a69854e2663083aeea1a0ff683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Wed, 2 Dec 2015 02:02:09 +0100 Subject: [PATCH 013/182] Avoid server from blocking if an exception is found in the main thread --- testserver/server.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testserver/server.py b/testserver/server.py index 5c5d8069..fd6bca40 100644 --- a/testserver/server.py +++ b/testserver/server.py @@ -69,5 +69,10 @@ class Server(threading.Thread): def __exit__(self, exc_type, exc_value, traceback): if exc_type is None: self.stop_event.wait() + else: + if self.wait_to_close_event: + # avoid server from blocking if an exception is found + # in the main thread + self.wait_to_close_event.set() return False # allow exceptions to propagate From 3b2a489e13a2dc3c9d1ef919ad35e14cc5189f10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Tue, 8 Dec 2015 16:12:40 +0100 Subject: [PATCH 014/182] Added text_response_server classmethod to TestServer and updated basic_response_server to use it --- testserver/server.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/testserver/server.py b/testserver/server.py index fd6bca40..8361b58f 100644 --- a/testserver/server.py +++ b/testserver/server.py @@ -23,17 +23,22 @@ class Server(threading.Thread): self.stop_event = threading.Event() @classmethod - def basic_response_server(cls, **kwargs): - def basic_response_handler(sock): - sock.send( - b'HTTP/1.1 200 OK\r\n' - b'Content-Length: 0\r\n' - b'\r\n' - ) + def text_response_server(cls, text, **kwargs): + def text_response_handler(sock): + sock.send(text.encode()) + + server = Server(text_response_handler, **kwargs) - server = Server(basic_response_handler, **kwargs) return server + @classmethod + def basic_response_server(cls, **kwargs): + server = cls.text_response_server( + "HTTP/1.1 200 OK\r\n" + + "Content-Length: 0\r\n\r\n", **kwargs + ) + + return server def run(self): try: From 4a41c591f88c3be7bb1319c8e6b8eba631fea8da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Tue, 8 Dec 2015 16:23:02 +0100 Subject: [PATCH 015/182] Tested text_response_server --- test_requests.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test_requests.py b/test_requests.py index 5c91c20a..29773bda 100755 --- a/test_requests.py +++ b/test_requests.py @@ -1784,6 +1784,20 @@ class TestTestServer(unittest.TestCase): with pytest.raises(socket.error): new_sock = socket.socket() new_sock.connect((host, port)) + + def test_text_response(self): + server = Server.text_response_server( + "HTTP/1.1 200 OK\r\n" + + "Content-Length: 6\r\n" + + "\r\nroflol" + ) + + with server as (host, port): + r = requests.get('http://{}:{}'.format(host, port)) + + assert r.status_code == 200 + assert r.text == 'roflol' + assert r.headers['Content-Length'] == '6' def test_basic_response(self): with Server.basic_response_server() as (host, port): From 578b03255594d446b4fc8c6e00ec67dd3c8b440f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Tue, 8 Dec 2015 16:37:03 +0100 Subject: [PATCH 016/182] Tested TestServer multiple request handling --- test_requests.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test_requests.py b/test_requests.py index 29773bda..aa591ef5 100755 --- a/test_requests.py +++ b/test_requests.py @@ -1817,5 +1817,22 @@ class TestTestServer(unittest.TestCase): sock.send(b'still alive') block_server.set() # release server block + def test_multiple_requests(self): + requests_to_handle = 5 + + server = Server.basic_response_server(requests_to_handle=requests_to_handle) + + with server as (host, port): + server_url = 'http://{}:{}'.format(host, port) + for _ in range(requests_to_handle): + r = requests.get(server_url) + assert r.status_code == 200 + + # the (n+1)th request fails + with pytest.raises(requests.exceptions.ConnectionError): + r = requests.get(server_url) + + + if __name__ == '__main__': unittest.main() From 3eeea3771b274e1089317faa302730e74203b37c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Tue, 8 Dec 2015 20:40:58 +0100 Subject: [PATCH 017/182] Added a way for TestServer request handlers to store data for later retrieval by the main thread (actual requests, for example) --- test_requests.py | 22 +++++++++++++++++++++- testserver/server.py | 33 ++++++++++++++++++++++++++++----- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/test_requests.py b/test_requests.py index aa591ef5..4e97bd66 100755 --- a/test_requests.py +++ b/test_requests.py @@ -1779,6 +1779,7 @@ class TestTestServer(unittest.TestCase): with Server.basic_response_server() as (host, port): sock = socket.socket() sock.connect((host, port)) + sock.close() with pytest.raises(socket.error): @@ -1797,7 +1798,7 @@ class TestTestServer(unittest.TestCase): assert r.status_code == 200 assert r.text == 'roflol' - assert r.headers['Content-Length'] == '6' + assert r.headers['Content-Length'] == '6' def test_basic_response(self): with Server.basic_response_server() as (host, port): @@ -1832,6 +1833,25 @@ class TestTestServer(unittest.TestCase): with pytest.raises(requests.exceptions.ConnectionError): r = requests.get(server_url) + def test_request_recovery(self): + server = Server.basic_response_server(requests_to_handle=2) + first_request = "put your hands up in the air" + second_request = "put your hand down in the floor" + + with server as address: + sock1 = socket.socket() + sock2 = socket.socket() + + sock1.connect(address) + sock1.send(first_request.encode()) + sock1.close() + + sock2.connect(address) + sock2.send(second_request.encode()) + sock2.close() + + assert server.handler_results[0] == first_request + assert server.handler_results[1] == second_request if __name__ == '__main__': diff --git a/testserver/server.py b/testserver/server.py index 8361b58f..c5a7ad5b 100644 --- a/testserver/server.py +++ b/testserver/server.py @@ -2,18 +2,35 @@ import threading import socket +import select -def consume_socket(sock, chunks=65536): - while not sock.recv(chunks).endswith(b'\r\n\r\n'): - pass +def consume_socket_content(sock, chunks=65536, timeout=0.5): + content = "" + more_to_read = select.select([sock], [], [], timeout)[0] + + while more_to_read: + new_content = sock.recv(chunks).decode("utf-8") + + if len(new_content) == 0: + more_to_read = False # empty recv means the socket disconnected + + else: + content += new_content + # stop reading if no new data is received for a while + more_to_read = select.select([sock], [], [], timeout)[0] + + return content class Server(threading.Thread): """ Dummy server using for unit testing """ def __init__(self, handler, host='localhost', port=0, requests_to_handle=1, wait_to_close_event=None): threading.Thread.__init__(self) + self.handler = handler + self.handler_results = [] + self.host = host self.port = port self.requests_to_handle = requests_to_handle @@ -23,10 +40,14 @@ class Server(threading.Thread): self.stop_event = threading.Event() @classmethod - def text_response_server(cls, text, **kwargs): + def text_response_server(cls, text, request_timeout=0.5, **kwargs): def text_response_handler(sock): + request_content = consume_socket_content(sock, timeout=request_timeout) sock.send(text.encode()) + return request_content + + server = Server(text_response_handler, **kwargs) return server @@ -61,7 +82,9 @@ class Server(threading.Thread): def _handle_requests_and_close_server(self, server_sock): for _ in range(self.requests_to_handle): sock = server_sock.accept()[0] - self.handler(sock) + handler_result = self.handler(sock) + + self.handler_results.append(handler_result) if self.wait_to_close_event: self.wait_to_close_event.wait() From 534ee814fcb9d7b563413ea77ce6838090ead25f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Braulio=20Valdivielso=20Mart=C3=ADnez?= Date: Tue, 8 Dec 2015 20:54:12 +0100 Subject: [PATCH 018/182] Tested timeouts in text_response_server requests retrieval --- test_requests.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test_requests.py b/test_requests.py index 4e97bd66..f24c775c 100755 --- a/test_requests.py +++ b/test_requests.py @@ -1853,6 +1853,31 @@ class TestTestServer(unittest.TestCase): assert server.handler_results[0] == first_request assert server.handler_results[1] == second_request + def test_requests_after_timeout_are_not_received(self): + server = Server.basic_response_server(request_timeout=1) + + with server as address: + sock = socket.socket() + sock.connect(address) + time.sleep(1.5) + sock.send(b"hehehe, not received") + sock.close() + + assert server.handler_results[0] == "" + + + def test_request_recovery_with_bigger_timeout(self): + server = Server.basic_response_server(request_timeout=3) + data = "bananadine" + + with server as address: + sock = socket.socket() + sock.connect(address) + time.sleep(1.5) + sock.send(data.encode()) + sock.close() + + assert server.handler_results[0] == data if __name__ == '__main__': unittest.main() From cd84b3ff943f733c2e76bb047ab2d3a60cac5ba2 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 30 Dec 2015 10:07:16 +0000 Subject: [PATCH 019/182] Update to urllib3 1.14 --- requests/packages/urllib3/__init__.py | 4 +- requests/packages/urllib3/connectionpool.py | 23 ++- .../packages/urllib3/contrib/appengine.py | 12 +- requests/packages/urllib3/contrib/ntlmpool.py | 20 +- requests/packages/urllib3/contrib/socks.py | 172 ++++++++++++++++++ requests/packages/urllib3/exceptions.py | 8 + requests/packages/urllib3/poolmanager.py | 17 +- requests/packages/urllib3/util/response.py | 2 +- requests/packages/urllib3/util/retry.py | 4 +- requests/packages/urllib3/util/ssl_.py | 3 +- 10 files changed, 228 insertions(+), 37 deletions(-) create mode 100644 requests/packages/urllib3/contrib/socks.py diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index e43991a9..b744315b 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -32,7 +32,7 @@ except ImportError: __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.13.1' +__version__ = '1.14' __all__ = ( 'HTTPConnectionPool', @@ -68,7 +68,7 @@ def add_stderr_logger(level=logging.DEBUG): handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) logger.addHandler(handler) logger.setLevel(level) - logger.debug('Added a stderr logging handler to logger: %s' % __name__) + logger.debug('Added a stderr logging handler to logger: %s', __name__) return handler # ... Clean up. diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 995b4167..01afd616 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -203,8 +203,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): Return a fresh :class:`HTTPConnection`. """ self.num_connections += 1 - log.info("Starting new HTTP connection (%d): %s" % - (self.num_connections, self.host)) + log.info("Starting new HTTP connection (%d): %s", + self.num_connections, self.host) conn = self.ConnectionCls(host=self.host, port=self.port, timeout=self.timeout.connect_timeout, @@ -239,7 +239,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # If this is a persistent connection, check if it got disconnected if conn and is_connection_dropped(conn): - log.info("Resetting dropped connection: %s" % self.host) + log.info("Resetting dropped connection: %s", self.host) conn.close() if getattr(conn, 'auto_open', 1) == 0: # This is a proxied connection that has been mutated by @@ -272,7 +272,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): except Full: # This should never happen if self.block == True log.warning( - "Connection pool is full, discarding connection: %s" % + "Connection pool is full, discarding connection: %s", self.host) # Connection never got put back into the pool, close it. @@ -382,9 +382,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # AppEngine doesn't have a version attr. http_version = getattr(conn, '_http_vsn_str', 'HTTP/?') - log.debug("\"%s %s %s\" %s %s" % (method, url, http_version, - httplib_response.status, - httplib_response.length)) + log.debug("\"%s %s %s\" %s %s", method, url, http_version, + httplib_response.status, httplib_response.length) try: assert_header_parsing(httplib_response.msg) @@ -622,7 +621,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if not conn: # Try again log.warning("Retrying (%r) after connection " - "broken by '%r': %s" % (retries, err, url)) + "broken by '%r': %s", retries, err, url) return self.urlopen(method, url, body, headers, retries, redirect, assert_same_host, timeout=timeout, pool_timeout=pool_timeout, @@ -644,7 +643,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): raise return response - log.info("Redirecting %s -> %s" % (url, redirect_location)) + log.info("Redirecting %s -> %s", url, redirect_location) return self.urlopen( method, redirect_location, body, headers, retries=retries, redirect=redirect, @@ -656,7 +655,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if retries.is_forced_retry(method, status_code=response.status): retries = retries.increment(method, url, response=response, _pool=self) retries.sleep() - log.info("Forced retry: %s" % url) + log.info("Forced retry: %s", url) return self.urlopen( method, url, body, headers, retries=retries, redirect=redirect, @@ -754,8 +753,8 @@ class HTTPSConnectionPool(HTTPConnectionPool): Return a fresh :class:`httplib.HTTPSConnection`. """ self.num_connections += 1 - log.info("Starting new HTTPS connection (%d): %s" - % (self.num_connections, self.host)) + log.info("Starting new HTTPS connection (%d): %s", + self.num_connections, self.host) if not self.ConnectionCls or self.ConnectionCls is DummyConnection: raise SSLError("Can't connect to HTTPS URL because the SSL " diff --git a/requests/packages/urllib3/contrib/appengine.py b/requests/packages/urllib3/contrib/appengine.py index 884cdb22..f4289c0f 100644 --- a/requests/packages/urllib3/contrib/appengine.py +++ b/requests/packages/urllib3/contrib/appengine.py @@ -144,7 +144,7 @@ class AppEngineManager(RequestMethods): if retries.is_forced_retry(method, status_code=http_response.status): retries = retries.increment( method, url, response=http_response, _pool=self) - log.info("Forced retry: %s" % url) + log.info("Forced retry: %s", url) retries.sleep() return self.urlopen( method, url, @@ -164,6 +164,14 @@ class AppEngineManager(RequestMethods): if content_encoding == 'deflate': del urlfetch_resp.headers['content-encoding'] + transfer_encoding = urlfetch_resp.headers.get('transfer-encoding') + # We have a full response's content, + # so let's make sure we don't report ourselves as chunked data. + if transfer_encoding == 'chunked': + encodings = transfer_encoding.split(",") + encodings.remove('chunked') + urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings) + return HTTPResponse( # In order for decoding to work, we must present the content as # a file-like object. @@ -177,7 +185,7 @@ class AppEngineManager(RequestMethods): if timeout is Timeout.DEFAULT_TIMEOUT: return 5 # 5s is the default timeout for URLFetch. if isinstance(timeout, Timeout): - if timeout.read is not timeout.connect: + if timeout._read is not timeout._connect: warnings.warn( "URLFetch does not support granular timeout settings, " "reverting to total timeout.", AppEnginePlatformWarning) diff --git a/requests/packages/urllib3/contrib/ntlmpool.py b/requests/packages/urllib3/contrib/ntlmpool.py index c136a238..11d0b5c3 100644 --- a/requests/packages/urllib3/contrib/ntlmpool.py +++ b/requests/packages/urllib3/contrib/ntlmpool.py @@ -43,8 +43,8 @@ class NTLMConnectionPool(HTTPSConnectionPool): # Performs the NTLM handshake that secures the connection. The socket # must be kept open while requests are performed. self.num_connections += 1 - log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s' % - (self.num_connections, self.host, self.authurl)) + log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s', + self.num_connections, self.host, self.authurl) headers = {} headers['Connection'] = 'Keep-Alive' @@ -56,13 +56,13 @@ class NTLMConnectionPool(HTTPSConnectionPool): # Send negotiation message headers[req_header] = ( 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)) - log.debug('Request headers: %s' % headers) + log.debug('Request headers: %s', headers) conn.request('GET', self.authurl, None, headers) res = conn.getresponse() reshdr = dict(res.getheaders()) - log.debug('Response status: %s %s' % (res.status, res.reason)) - log.debug('Response headers: %s' % reshdr) - log.debug('Response data: %s [...]' % res.read(100)) + log.debug('Response status: %s %s', res.status, res.reason) + log.debug('Response headers: %s', reshdr) + log.debug('Response data: %s [...]', res.read(100)) # Remove the reference to the socket, so that it can not be closed by # the response object (we want to keep the socket open) @@ -87,12 +87,12 @@ class NTLMConnectionPool(HTTPSConnectionPool): self.pw, NegotiateFlags) headers[req_header] = 'NTLM %s' % auth_msg - log.debug('Request headers: %s' % headers) + log.debug('Request headers: %s', headers) conn.request('GET', self.authurl, None, headers) res = conn.getresponse() - log.debug('Response status: %s %s' % (res.status, res.reason)) - log.debug('Response headers: %s' % dict(res.getheaders())) - log.debug('Response data: %s [...]' % res.read()[:100]) + log.debug('Response status: %s %s', res.status, res.reason) + log.debug('Response headers: %s', dict(res.getheaders())) + log.debug('Response data: %s [...]', res.read()[:100]) if res.status != 200: if res.status == 401: raise Exception('Server rejected request: wrong ' diff --git a/requests/packages/urllib3/contrib/socks.py b/requests/packages/urllib3/contrib/socks.py new file mode 100644 index 00000000..885776c3 --- /dev/null +++ b/requests/packages/urllib3/contrib/socks.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +""" +SOCKS support for urllib3 +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This contrib module contains provisional support for SOCKS proxies from within +urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and +SOCKS5. To enable its functionality, either install PySocks or install this +module with the ``socks`` extra. + +Known Limitations: + +- Currently PySocks does not support contacting remote websites via literal + IPv6 addresses. Any such connection attempt will fail. +- Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any + such connection attempt will fail. +""" +from __future__ import absolute_import + +try: + import socks +except ImportError: + import warnings + from urllib3.exceptions import DependencyWarning + + warnings.warn(( + 'SOCKS support in urllib3 requires the installation of optional ' + 'dependencies: specifically, PySocks. For more information, see ' + 'https://urllib3.readthedocs.org/en/latest/contrib.html#socks-proxies' + ), + DependencyWarning + ) + raise + +from socket import error as SocketError, timeout as SocketTimeout + +from urllib3.connection import ( + HTTPConnection, HTTPSConnection +) +from urllib3.connectionpool import ( + HTTPConnectionPool, HTTPSConnectionPool +) +from urllib3.exceptions import ConnectTimeoutError, NewConnectionError +from urllib3.poolmanager import PoolManager +from urllib3.util.url import parse_url + +try: + import ssl +except ImportError: + ssl = None + + +class SOCKSConnection(HTTPConnection): + """ + A plain-text HTTP connection that connects via a SOCKS proxy. + """ + def __init__(self, *args, **kwargs): + self._socks_options = kwargs.pop('_socks_options') + super(SOCKSConnection, self).__init__(*args, **kwargs) + + def _new_conn(self): + """ + Establish a new connection via the SOCKS proxy. + """ + extra_kw = {} + if self.source_address: + extra_kw['source_address'] = self.source_address + + if self.socket_options: + extra_kw['socket_options'] = self.socket_options + + try: + conn = socks.create_connection( + (self.host, self.port), + proxy_type=self._socks_options['socks_version'], + proxy_addr=self._socks_options['proxy_host'], + proxy_port=self._socks_options['proxy_port'], + proxy_username=self._socks_options['username'], + proxy_password=self._socks_options['password'], + timeout=self.timeout, + **extra_kw + ) + + except SocketTimeout as e: + raise ConnectTimeoutError( + self, "Connection to %s timed out. (connect timeout=%s)" % + (self.host, self.timeout)) + + except socks.ProxyError as e: + # This is fragile as hell, but it seems to be the only way to raise + # useful errors here. + if e.socket_err: + error = e.socket_err + if isinstance(error, SocketTimeout): + raise ConnectTimeoutError( + self, + "Connection to %s timed out. (connect timeout=%s)" % + (self.host, self.timeout) + ) + else: + raise NewConnectionError( + self, + "Failed to establish a new connection: %s" % error + ) + else: + raise NewConnectionError( + self, + "Failed to establish a new connection: %s" % e + ) + + except SocketError as e: # Defensive: PySocks should catch all these. + raise NewConnectionError( + self, "Failed to establish a new connection: %s" % e) + + return conn + + +# We don't need to duplicate the Verified/Unverified distinction from +# urllib3/connection.py here because the HTTPSConnection will already have been +# correctly set to either the Verified or Unverified form by that module. This +# means the SOCKSHTTPSConnection will automatically be the correct type. +class SOCKSHTTPSConnection(SOCKSConnection, HTTPSConnection): + pass + + +class SOCKSHTTPConnectionPool(HTTPConnectionPool): + ConnectionCls = SOCKSConnection + + +class SOCKSHTTPSConnectionPool(HTTPSConnectionPool): + ConnectionCls = SOCKSHTTPSConnection + + +class SOCKSProxyManager(PoolManager): + """ + A version of the urllib3 ProxyManager that routes connections via the + defined SOCKS proxy. + """ + pool_classes_by_scheme = { + 'http': SOCKSHTTPConnectionPool, + 'https': SOCKSHTTPSConnectionPool, + } + + def __init__(self, proxy_url, username=None, password=None, + num_pools=10, headers=None, **connection_pool_kw): + parsed = parse_url(proxy_url) + + if parsed.scheme == 'socks5': + socks_version = socks.PROXY_TYPE_SOCKS5 + elif parsed.scheme == 'socks4': + socks_version = socks.PROXY_TYPE_SOCKS4 + else: + raise ValueError( + "Unable to determine SOCKS version from %s" % proxy_url + ) + + self.proxy_url = proxy_url + + socks_options = { + 'socks_version': socks_version, + 'proxy_host': parsed.host, + 'proxy_port': parsed.port, + 'username': username, + 'password': password, + } + connection_pool_kw['_socks_options'] = socks_options + + super(SOCKSProxyManager, self).__init__( + num_pools, headers, **connection_pool_kw + ) + + self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py index 8e07eb61..f2e65917 100644 --- a/requests/packages/urllib3/exceptions.py +++ b/requests/packages/urllib3/exceptions.py @@ -180,6 +180,14 @@ class SNIMissingWarning(HTTPWarning): pass +class DependencyWarning(HTTPWarning): + """ + Warned when an attempt is made to import a module with missing optional + dependencies. + """ + pass + + class ResponseNotChunked(ProtocolError, ValueError): "Response needs to be chunked in order to read it as chunks." pass diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index f13e673d..1023dcba 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -18,16 +18,16 @@ from .util.retry import Retry __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] -pool_classes_by_scheme = { - 'http': HTTPConnectionPool, - 'https': HTTPSConnectionPool, -} - log = logging.getLogger(__name__) SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', 'ssl_version', 'ca_cert_dir') +pool_classes_by_scheme = { + 'http': HTTPConnectionPool, + 'https': HTTPSConnectionPool, +} + class PoolManager(RequestMethods): """ @@ -65,6 +65,9 @@ class PoolManager(RequestMethods): self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close()) + # Locally set the pool classes so other PoolManagers can override them. + self.pool_classes_by_scheme = pool_classes_by_scheme + def __enter__(self): return self @@ -81,7 +84,7 @@ class PoolManager(RequestMethods): by :meth:`connection_from_url` and companion methods. It is intended to be overridden for customization. """ - pool_cls = pool_classes_by_scheme[scheme] + pool_cls = self.pool_classes_by_scheme[scheme] kwargs = self.connection_pool_kw if scheme == 'http': kwargs = self.connection_pool_kw.copy() @@ -186,7 +189,7 @@ class PoolManager(RequestMethods): kw['retries'] = retries kw['redirect'] = redirect - log.info("Redirecting %s -> %s" % (url, redirect_location)) + log.info("Redirecting %s -> %s", url, redirect_location) return self.urlopen(method, redirect_location, **kw) diff --git a/requests/packages/urllib3/util/response.py b/requests/packages/urllib3/util/response.py index bc723272..0b5c75c1 100644 --- a/requests/packages/urllib3/util/response.py +++ b/requests/packages/urllib3/util/response.py @@ -61,7 +61,7 @@ def assert_header_parsing(headers): def is_response_to_head(response): """ - Checks, wether a the request of a response has been a HEAD-request. + Checks whether the request of a response has been a HEAD-request. Handles the quirks of AppEngine. :param conn: diff --git a/requests/packages/urllib3/util/retry.py b/requests/packages/urllib3/util/retry.py index 03a01249..862174ab 100644 --- a/requests/packages/urllib3/util/retry.py +++ b/requests/packages/urllib3/util/retry.py @@ -153,7 +153,7 @@ class Retry(object): redirect = bool(redirect) and None new_retries = cls(retries, redirect=redirect) - log.debug("Converted retries value: %r -> %r" % (retries, new_retries)) + log.debug("Converted retries value: %r -> %r", retries, new_retries) return new_retries def get_backoff_time(self): @@ -272,7 +272,7 @@ class Retry(object): if new_retry.is_exhausted(): raise MaxRetryError(_pool, url, error or ResponseError(cause)) - log.debug("Incremented Retry for (url='%s'): %r" % (url, new_retry)) + log.debug("Incremented Retry for (url='%s'): %r", url, new_retry) return new_retry diff --git a/requests/packages/urllib3/util/ssl_.py b/requests/packages/urllib3/util/ssl_.py index 67f83441..313d3a7e 100644 --- a/requests/packages/urllib3/util/ssl_.py +++ b/requests/packages/urllib3/util/ssl_.py @@ -110,7 +110,7 @@ except ImportError: ) self.ciphers = cipher_suite - def wrap_socket(self, socket, server_hostname=None): + def wrap_socket(self, socket, server_hostname=None, server_side=False): warnings.warn( 'A true SSLContext object is not available. This prevents ' 'urllib3 from configuring SSL appropriately and may cause ' @@ -125,6 +125,7 @@ except ImportError: 'ca_certs': self.ca_certs, 'cert_reqs': self.verify_mode, 'ssl_version': self.protocol, + 'server_side': server_side, } if self.supports_set_ciphers: # Platform-specific: Python 2.7+ return wrap_socket(socket, ciphers=self.ciphers, **kwargs) From 396963ac7eb8078bae5c30d9cf596debef892c3d Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 30 Dec 2015 10:40:12 +0000 Subject: [PATCH 020/182] Temporary urllib3 patch --- requests/packages/urllib3/__init__.py | 3 +-- requests/packages/urllib3/contrib/socks.py | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index b744315b..a5ad1332 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -1,7 +1,6 @@ """ urllib3 - Thread-safe connection pooling and re-using. """ - from __future__ import absolute_import import warnings @@ -32,7 +31,7 @@ except ImportError: __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.14' +__version__ = 'dev' __all__ = ( 'HTTPConnectionPool', diff --git a/requests/packages/urllib3/contrib/socks.py b/requests/packages/urllib3/contrib/socks.py index 885776c3..3748fee5 100644 --- a/requests/packages/urllib3/contrib/socks.py +++ b/requests/packages/urllib3/contrib/socks.py @@ -21,7 +21,7 @@ try: import socks except ImportError: import warnings - from urllib3.exceptions import DependencyWarning + from ..exceptions import DependencyWarning warnings.warn(( 'SOCKS support in urllib3 requires the installation of optional ' @@ -34,15 +34,15 @@ except ImportError: from socket import error as SocketError, timeout as SocketTimeout -from urllib3.connection import ( +from ..connection import ( HTTPConnection, HTTPSConnection ) -from urllib3.connectionpool import ( +from ..connectionpool import ( HTTPConnectionPool, HTTPSConnectionPool ) -from urllib3.exceptions import ConnectTimeoutError, NewConnectionError -from urllib3.poolmanager import PoolManager -from urllib3.util.url import parse_url +from ..exceptions import ConnectTimeoutError, NewConnectionError +from ..poolmanager import PoolManager +from ..util.url import parse_url try: import ssl From 08364c6739ef94d245327acf88359de371bbf3ba Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 30 Dec 2015 10:40:24 +0000 Subject: [PATCH 021/182] Add support for SOCKS --- requests/adapters.py | 43 +++++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 6266d5be..521489a5 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -33,9 +33,15 @@ from .packages.urllib3.exceptions import SSLError as _SSLError from .packages.urllib3.exceptions import ResponseError from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError) + ProxyError, RetryError, InvalidSchema) from .auth import _basic_auth_str +try: + from .packages.urllib3.contrib.socks import SOCKSProxyManager +except ImportError: + def SOCKSProxyManager(*args, **kwargs): + raise InvalidSchema("Missing dependencies for SOCKS support.") + DEFAULT_POOLBLOCK = False DEFAULT_POOLSIZE = 10 DEFAULT_RETRIES = 0 @@ -150,14 +156,26 @@ class HTTPAdapter(BaseAdapter): :returns: ProxyManager """ if not proxy in self.proxy_manager: - proxy_headers = self.proxy_headers(proxy) - self.proxy_manager[proxy] = proxy_from_url( - proxy, - proxy_headers=proxy_headers, - num_pools=self._pool_connections, - maxsize=self._pool_maxsize, - block=self._pool_block, - **proxy_kwargs) + if proxy.lower().startswith('socks'): + username, password = get_auth_from_url(proxy) + self.proxy_manager[proxy] = SOCKSProxyManager( + proxy, + username=username, + password=password, + num_pools=self._pool_connections, + maxsize=self._pool_maxsize, + block=self._pool_block, + **proxy_kwargs + ) + else: + proxy_headers = self.proxy_headers(proxy) + self.proxy_manager[proxy] = proxy_from_url( + proxy, + proxy_headers=proxy_headers, + num_pools=self._pool_connections, + maxsize=self._pool_maxsize, + block=self._pool_block, + **proxy_kwargs) return self.proxy_manager[proxy] @@ -284,7 +302,12 @@ class HTTPAdapter(BaseAdapter): """ proxy = select_proxy(request.url, proxies) scheme = urlparse(request.url).scheme - if proxy and scheme != 'https': + proxy_scheme = urlparse(proxy).scheme + + if proxy and proxy_scheme.lower().startswith('socks'): + # Socks proxies behave like the proxy isn't there at all. + url = request.path_url + elif proxy and scheme != 'https': url = urldefragauth(request.url) else: url = request.path_url From 39ee5d46dc05edec6d4cf9c2831b326b857fc0e4 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 30 Dec 2015 10:40:30 +0000 Subject: [PATCH 022/182] Silence SOCKS dependency warning on import --- requests/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/requests/__init__.py b/requests/__init__.py index bd5b5b97..39a1f38a 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -55,6 +55,12 @@ try: except ImportError: pass +import warnings + +# urllib3's DependencyWarnings should be silenced. +from .packages.urllib3.exceptions import DependencyWarning +warnings.simplefilter('ignore', DependencyWarning) + from . import utils from .models import Request, Response, PreparedRequest from .api import request, get, head, post, patch, put, delete, options From 1c54014daff8fa237820d5c67dcbd1aa2f585927 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 30 Dec 2015 10:48:29 +0000 Subject: [PATCH 023/182] Don't parse nonexistent URLs. --- requests/adapters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requests/adapters.py b/requests/adapters.py index 521489a5..7ca71775 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -302,7 +302,8 @@ class HTTPAdapter(BaseAdapter): """ proxy = select_proxy(request.url, proxies) scheme = urlparse(request.url).scheme - proxy_scheme = urlparse(proxy).scheme + if proxy: + proxy_scheme = urlparse(proxy).scheme if proxy and proxy_scheme.lower().startswith('socks'): # Socks proxies behave like the proxy isn't there at all. From 11b9b24ac9c6d7e2231493b3d8723de7715b0f2e Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 30 Dec 2015 14:37:53 +0000 Subject: [PATCH 024/182] Some code cleanups. --- requests/adapters.py | 48 +++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 7ca71775..6098ccdd 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -19,7 +19,7 @@ from .packages.urllib3.util.retry import Retry from .compat import urlparse, basestring from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers, prepend_scheme_if_needed, get_auth_from_url, urldefragauth, - select_proxy) + select_proxy, to_native_string) from .structures import CaseInsensitiveDict from .packages.urllib3.exceptions import ClosedPoolError from .packages.urllib3.exceptions import ConnectTimeoutError @@ -155,29 +155,30 @@ class HTTPAdapter(BaseAdapter): :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager. :returns: ProxyManager """ - if not proxy in self.proxy_manager: - if proxy.lower().startswith('socks'): - username, password = get_auth_from_url(proxy) - self.proxy_manager[proxy] = SOCKSProxyManager( - proxy, - username=username, - password=password, - num_pools=self._pool_connections, - maxsize=self._pool_maxsize, - block=self._pool_block, - **proxy_kwargs - ) - else: - proxy_headers = self.proxy_headers(proxy) - self.proxy_manager[proxy] = proxy_from_url( - proxy, - proxy_headers=proxy_headers, - num_pools=self._pool_connections, - maxsize=self._pool_maxsize, - block=self._pool_block, - **proxy_kwargs) + if proxy in self.proxy_manager: + manager = self.proxy_manager[proxy] + elif proxy.lower().startswith('socks'): + username, password = get_auth_from_url(proxy) + manager = self.proxy_manager[proxy] = SOCKSProxyManager( + proxy, + username=username, + password=password, + num_pools=self._pool_connections, + maxsize=self._pool_maxsize, + block=self._pool_block, + **proxy_kwargs + ) + else: + proxy_headers = self.proxy_headers(proxy) + manager = self.proxy_manager[proxy] = proxy_from_url( + proxy, + proxy_headers=proxy_headers, + num_pools=self._pool_connections, + maxsize=self._pool_maxsize, + block=self._pool_block, + **proxy_kwargs) - return self.proxy_manager[proxy] + return manager def cert_verify(self, conn, url, verify, cert): """Verify a SSL certificate. This method should not be called from user @@ -302,6 +303,7 @@ class HTTPAdapter(BaseAdapter): """ proxy = select_proxy(request.url, proxies) scheme = urlparse(request.url).scheme + proxy_scheme = '' if proxy: proxy_scheme = urlparse(proxy).scheme From e4d15c464929e87a6648c57f32c422dc5bfecef3 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 30 Dec 2015 16:45:51 +0000 Subject: [PATCH 025/182] Help @sigmavirus24 with code golf. --- requests/adapters.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 6098ccdd..18900f15 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -303,17 +303,16 @@ class HTTPAdapter(BaseAdapter): """ proxy = select_proxy(request.url, proxies) scheme = urlparse(request.url).scheme - proxy_scheme = '' - if proxy: - proxy_scheme = urlparse(proxy).scheme - if proxy and proxy_scheme.lower().startswith('socks'): - # Socks proxies behave like the proxy isn't there at all. - url = request.path_url - elif proxy and scheme != 'https': + is_proxied_http_request = (proxy and scheme != 'https') + using_socks_proxy = False + if proxy: + proxy_scheme = urlparse(proxy).scheme.lower() + using_socks_proxy = proxy_scheme.startswith('socks') + + url = request.path_url + if is_proxied_http_request and not using_socks_proxy: url = urldefragauth(request.url) - else: - url = request.path_url return url From a3acd30bcc284f02e5740b7ce9b4aa415af00bf5 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Thu, 31 Dec 2015 09:19:45 +0000 Subject: [PATCH 026/182] Socks extra --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index b7ed12ba..9e9c7232 100755 --- a/setup.py +++ b/setup.py @@ -70,5 +70,6 @@ setup( ), extras_require={ 'security': ['pyOpenSSL>=0.13', 'ndg-httpsclient', 'pyasn1'], + 'socks': ['PySocks>=1.5.6'], }, ) From b1e97650f7e1bed302246e08ff1eea26ef832fda Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 04:00:14 -0500 Subject: [PATCH 027/182] Update README.rst --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 99d30e72..5ad9c66b 100644 --- a/README.rst +++ b/README.rst @@ -15,7 +15,7 @@ beings. Most existing Python modules for sending HTTP requests are extremely verbose and cumbersome. Python's builtin urllib2 module provides most of -the HTTP capabilities you should need, but the api is thoroughly broken. +the HTTP capabilities you should need, but the API is thoroughly broken. It requires an enormous amount of work (even method overrides) to perform the simplest of tasks. @@ -33,8 +33,8 @@ Things shouldn't be this way. Not in Python. See `the same code, without Requests `_. -Requests allow you to send HTTP/1.1 requests. You can add headers, form data, -multipart files, and parameters with simple Python dictionaries, and access the +Requests allows you to send HTTP/1.1 requests. You can add headers, form data, +multi-part files, and parameters with simple Python dictionaries, and access the response data in the same way. It's powered by httplib and `urllib3 `_, but it does all the hard work and crazy hacks for you. From 20b4bb7c363822f6706522a1d2c018a094d32fb7 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 04:52:50 -0500 Subject: [PATCH 028/182] revised readme --- README.rst | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/README.rst b/README.rst index 99d30e72..5a289230 100644 --- a/README.rst +++ b/README.rst @@ -7,41 +7,39 @@ Requests: HTTP for Humans .. image:: https://img.shields.io/pypi/dm/requests.svg :target: https://pypi.python.org/pypi/requests +**Requests** is the only *Non-GMO* HTTP library for Python, safe for human +consumption. - - -Requests is an Apache2 Licensed HTTP library, written in Python, for human -beings. - -Most existing Python modules for sending HTTP requests are extremely -verbose and cumbersome. Python's builtin urllib2 module provides most of -the HTTP capabilities you should need, but the api is thoroughly broken. -It requires an enormous amount of work (even method overrides) to -perform the simplest of tasks. - -Things shouldn't be this way. Not in Python. +Consumption of other HTTP libraries may result in dangerous side-effects, +including: security vulnerabilities, verbose code, reinventing the wheel, +constantly reading documentation, depression, headaches, or even death. .. code-block:: python >>> r = requests.get('https://api.github.com', auth=('user', 'pass')) >>> r.status_code - 204 + 200 >>> r.headers['content-type'] - 'application/json' + 'application/json; charset=utf8' + >>> r.encoding + 'utf-8' >>> r.text - ... + u'{"type":"User"...' + >>> r.json() + {u'disk_usage': 368627, u'private_gists': 484, ...} -See `the same code, without Requests `_. +See `the same code, before Requests `_. -Requests allow you to send HTTP/1.1 requests. You can add headers, form data, -multipart files, and parameters with simple Python dictionaries, and access the -response data in the same way. It's powered by httplib and `urllib3 -`_, but it does all the hard work and crazy -hacks for you. +Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, very well. +Headers, cookies, json/form data, multipart files, and url parameters can all +be provided with simple Python dictionaries. SSL certificates are automatically +verified against a carefully crafted bundle of root certificates, included. +Responses feature RFC-compliant redirection history, unicode/bytes> bodies, a +case-insensitive header dictionary, and more. -Features --------- +Special Features +---------------- - International Domains and URLs - Keep-Alive & Connection Pooling @@ -50,12 +48,14 @@ Features - Basic/Digest Authentication - Elegant Key/Value Cookies - Automatic Decompression +- Automatic Content Decoding - Unicode Response Bodies - Multipart File Uploads -- Connection Timeouts -- Thread-safety - HTTP(S) proxy support - +- Connection Timeouts +- Streaming Downloads +- Chunked Requests +- Thread-safety Installation ------------ From 4095b26f136021ec93f44c3ac099ae90894fd783 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 04:58:22 -0500 Subject: [PATCH 029/182] readme cleanup --- README.rst | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index bed9c197..74cf991f 100644 --- a/README.rst +++ b/README.rst @@ -10,19 +10,13 @@ Requests: HTTP for Humans **Requests** is the only *Non-GMO* HTTP library for Python, safe for human beings. -Most existing Python modules for sending HTTP requests are extremely -verbose and cumbersome. Python's builtin urllib2 module provides most of -the HTTP capabilities you should need, but the API is thoroughly broken. -It requires an enormous amount of work (even method overrides) to -consumption. - Consumption of other HTTP libraries may result in dangerous side-effects, including: security vulnerabilities, verbose code, reinventing the wheel, constantly reading documentation, depression, headaches, or even death. .. code-block:: python - >>> r = requests.get('https://api.github.com', auth=('user', 'pass')) + >>> r = requests.get('https://api.github.com/user', auth=('user', 'pass')) >>> r.status_code 200 >>> r.headers['content-type'] @@ -34,13 +28,13 @@ constantly reading documentation, depression, headaches, or even death. >>> r.json() {u'disk_usage': 368627, u'private_gists': 484, ...} -See `the same code, before Requests `_. +See `the similar code, sans Requests `_. Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, very well. Headers, cookies, json/form data, multipart files, and url parameters can all be provided with simple Python dictionaries. SSL certificates are automatically verified against a carefully crafted bundle of root certificates, included. -Responses feature RFC-compliant redirection history, unicode/bytes> bodies, a +Responses feature RFC-compliant redirection history, unicode/bytes bodies, a case-insensitive header dictionary, and more. From 8c963b52ea90c2cdb4e27d7d576d6e2968cb0131 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 04:59:42 -0500 Subject: [PATCH 030/182] standard --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 74cf991f..bb8003df 100644 --- a/README.rst +++ b/README.rst @@ -32,7 +32,7 @@ See `the similar code, sans Requests `_. Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, very well. Headers, cookies, json/form data, multipart files, and url parameters can all -be provided with simple Python dictionaries. SSL certificates are automatically +be provided with standard Python dictionaries. SSL certificates are automatically verified against a carefully crafted bundle of root certificates, included. Responses feature RFC-compliant redirection history, unicode/bytes bodies, a case-insensitive header dictionary, and more. From f997c0590fb84481e4d33e2f3b45320e54466264 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 05:02:06 -0500 Subject: [PATCH 031/182] Update README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index bb8003df..967565c6 100644 --- a/README.rst +++ b/README.rst @@ -7,10 +7,10 @@ Requests: HTTP for Humans .. image:: https://img.shields.io/pypi/dm/requests.svg :target: https://pypi.python.org/pypi/requests -**Requests** is the only *Non-GMO* HTTP library for Python, safe for human +Requests is the only *Non-GMO* HTTP library for Python, safe for human beings. -Consumption of other HTTP libraries may result in dangerous side-effects, +**Warning:** Consumption of other HTTP libraries may result in dangerous side-effects, including: security vulnerabilities, verbose code, reinventing the wheel, constantly reading documentation, depression, headaches, or even death. From 52054be6125efdb44a073fa6d62a81c8cd301129 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 05:04:23 -0500 Subject: [PATCH 032/182] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 967565c6..f92bd540 100644 --- a/README.rst +++ b/README.rst @@ -30,7 +30,7 @@ constantly reading documentation, depression, headaches, or even death. See `the similar code, sans Requests `_. -Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, very well. +Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, all day. Headers, cookies, json/form data, multipart files, and url parameters can all be provided with standard Python dictionaries. SSL certificates are automatically verified against a carefully crafted bundle of root certificates, included. From c0037973198bcb26788d167fb77de8ec2ee98a60 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 05:05:48 -0500 Subject: [PATCH 033/182] Update README.rst --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index f92bd540..f6df6d8f 100644 --- a/README.rst +++ b/README.rst @@ -32,8 +32,7 @@ See `the similar code, sans Requests `_. Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, all day. Headers, cookies, json/form data, multipart files, and url parameters can all -be provided with standard Python dictionaries. SSL certificates are automatically -verified against a carefully crafted bundle of root certificates, included. +be provided with standard Python dictionaries. Responses feature RFC-compliant redirection history, unicode/bytes bodies, a case-insensitive header dictionary, and more. From ab22b3ea5fc7340eb8daf21f45974372030a1b7d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 05:07:08 -0500 Subject: [PATCH 034/182] Update README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index f6df6d8f..0151714c 100644 --- a/README.rst +++ b/README.rst @@ -8,9 +8,9 @@ Requests: HTTP for Humans :target: https://pypi.python.org/pypi/requests Requests is the only *Non-GMO* HTTP library for Python, safe for human -beings. +consumption. -**Warning:** Consumption of other HTTP libraries may result in dangerous side-effects, +**Warning:** Recreational use of other HTTP libraries may result in dangerous side-effects, including: security vulnerabilities, verbose code, reinventing the wheel, constantly reading documentation, depression, headaches, or even death. From 33d2ba961bb8c39d307476851f50d26eab08026b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 05:07:33 -0500 Subject: [PATCH 035/182] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 0151714c..2e0fd80a 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,7 @@ Requests: HTTP for Humans Requests is the only *Non-GMO* HTTP library for Python, safe for human consumption. -**Warning:** Recreational use of other HTTP libraries may result in dangerous side-effects, +**Warning:** recreational use of other HTTP libraries may result in dangerous side-effects, including: security vulnerabilities, verbose code, reinventing the wheel, constantly reading documentation, depression, headaches, or even death. From 72d8293e0837fe29740a294b9b6d38a1b73172d4 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 05:07:47 -0500 Subject: [PATCH 036/182] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 2e0fd80a..0151714c 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,7 @@ Requests: HTTP for Humans Requests is the only *Non-GMO* HTTP library for Python, safe for human consumption. -**Warning:** recreational use of other HTTP libraries may result in dangerous side-effects, +**Warning:** Recreational use of other HTTP libraries may result in dangerous side-effects, including: security vulnerabilities, verbose code, reinventing the wheel, constantly reading documentation, depression, headaches, or even death. From 9c939ec026281dc5efcda5f52fde367cf2658b2a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 05:09:25 -0500 Subject: [PATCH 037/182] Update README.rst --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 0151714c..1986ddea 100644 --- a/README.rst +++ b/README.rst @@ -65,6 +65,7 @@ To install Requests, simply: $ pip install requests +Satisfaction, gaurenteed. Documentation ------------- From 9057f7539b9341ba3c739eb87af09995828f9544 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 05:09:45 -0500 Subject: [PATCH 038/182] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 1986ddea..485360d0 100644 --- a/README.rst +++ b/README.rst @@ -70,7 +70,7 @@ Satisfaction, gaurenteed. Documentation ------------- -Documentation is available at http://docs.python-requests.org/. +Excellent documentation is available at http://docs.python-requests.org/. Contribute From 4d97b5b39a7a461ae8942970c0de48062c2c839f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 05:13:34 -0500 Subject: [PATCH 039/182] Update README.rst --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 485360d0..34bb02b4 100644 --- a/README.rst +++ b/README.rst @@ -70,11 +70,11 @@ Satisfaction, gaurenteed. Documentation ------------- -Excellent documentation is available at http://docs.python-requests.org/. +Fantastic documentation is available at http://docs.python-requests.org/, for a limited time only. -Contribute ----------- +How to Contribute +----------------- #. Check for open issues or open a fresh issue to start a discussion around a feature idea or a bug. There is a `Contributor Friendly`_ tag for issues that should be ideal for people who are not very familiar with the codebase yet. #. Fork `the repository`_ on GitHub to start making your changes to the **master** branch (or branch off of it). From fbe104876efe9c8a792c12fe089f749edf0f59ee Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 05:14:01 -0500 Subject: [PATCH 040/182] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 34bb02b4..dfbaab82 100644 --- a/README.rst +++ b/README.rst @@ -65,7 +65,7 @@ To install Requests, simply: $ pip install requests -Satisfaction, gaurenteed. +Satisfaction, guaranteed. Documentation ------------- From ba2b355168f3ddfce8e155011b38423c0d839e11 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 05:33:23 -0500 Subject: [PATCH 041/182] Update README.rst --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index dfbaab82..268b9ee6 100644 --- a/README.rst +++ b/README.rst @@ -36,6 +36,9 @@ be provided with standard Python dictionaries. Responses feature RFC-compliant redirection history, unicode/bytes bodies, a case-insensitive header dictionary, and more. +Besides, all the cool kids are doing it. Requests is one of the most +downloaded Python packages of all time, pulling in over 6,000,000 downloads +every month. You don't want to be left out! Special Features ---------------- From 54d3b59f8914ac2b5a543b05210f22283c12281c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 05:50:45 -0500 Subject: [PATCH 042/182] update --- README.rst | 30 ++++++++++++++++++++---------- docs/index.rst | 48 +++++++++++++++++++++++++++++------------------- 2 files changed, 49 insertions(+), 29 deletions(-) diff --git a/README.rst b/README.rst index 268b9ee6..2fe6a85f 100644 --- a/README.rst +++ b/README.rst @@ -30,18 +30,25 @@ constantly reading documentation, depression, headaches, or even death. See `the similar code, sans Requests `_. -Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, all day. -Headers, cookies, json/form data, multipart files, and url parameters can all -be provided with standard Python dictionaries. +Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, without the +need for manual labor. There's no need to manually add query strings to your +URLs, or to form-encode your POST data. Keep-alive and HTTP connection pooling +are 100% automatic, powered by `urllib3 `_, +which is embedded within Requests. + +But wait, there's more! Headers, cookies, json/form data, multipart files, and +url parameters can all be provided with standard Python dictionaries. Responses feature RFC-compliant redirection history, unicode/bytes bodies, a case-insensitive header dictionary, and more. -Besides, all the cool kids are doing it. Requests is one of the most -downloaded Python packages of all time, pulling in over 6,000,000 downloads -every month. You don't want to be left out! +Besides, all the cool kids are doing it. Requests is one of the most +downloaded Python packages of all time, pulling in over 6,000,000 downloads +every month. You don't want to be left out! -Special Features ----------------- +Feature Support +--------------- + +Requests is ready for today's web. - International Domains and URLs - Keep-Alive & Connection Pooling @@ -53,12 +60,15 @@ Special Features - Automatic Content Decoding - Unicode Response Bodies - Multipart File Uploads -- HTTP(S) proxy support +- HTTP(S) Proxy Support - Connection Timeouts - Streaming Downloads +- ``.netrc`` Support - Chunked Requests - Thread-safety +Requests supports Python 2.6 — 3.5, and runs great on PyPy. + Installation ------------ @@ -68,7 +78,7 @@ To install Requests, simply: $ pip install requests -Satisfaction, guaranteed. +Satisfaction, guaranteed. Documentation ------------- diff --git a/docs/index.rst b/docs/index.rst index 617e01a3..5dccd3f7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,16 +8,12 @@ Requests: HTTP for Humans Release v\ |version|. (:ref:`Installation `) -Requests is an :ref:`Apache2 Licensed ` HTTP library, written in -Python, for human beings. +Requests is the only *Non-GMO* HTTP library for Python, safe for human +consumption. -Python's standard **urllib2** module provides most of -the HTTP capabilities you need, but the API is thoroughly **broken**. -It was built for a different time — and a different web. It requires an -*enormous* amount of work (even method overrides) to perform the simplest of -tasks. - -Things shouldn’t be this way. Not in Python. +**Warning:** Recreational use of other HTTP libraries may result in dangerous side-effects, +including: security vulnerabilities, verbose code, reinventing the wheel, +constantly reading documentation, depression, headaches, or even death. :: @@ -33,22 +29,31 @@ Things shouldn’t be this way. Not in Python. >>> r.json() {u'private_gists': 419, u'total_private_repos': 77, ...} -See `similar code, without Requests `_. +See `similar code, sans Requests `_. -Requests takes all of the work out of Python HTTP/1.1 — making your integration -with web services seamless. There's no need to manually add query strings to -your URLs, or to form-encode your POST data. Keep-alive and HTTP connection -pooling are 100% automatic, powered by `urllib3 `_, + +Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, without the +need for manual labor. There's no need to manually add query strings to your +URLs, or to form-encode your POST data. Keep-alive and HTTP connection pooling +are 100% automatic, powered by `urllib3 `_, which is embedded within Requests. +But wait, there's more! Headers, cookies, json/form data, multipart files, and +url parameters can all be provided with standard Python dictionaries. +Responses feature RFC-compliant redirection history, unicode/bytes bodies, a +case-insensitive header dictionary, and more. + +Besides, all the cool kids are doing it. Requests is one of the most +downloaded Python packages of all time, pulling in over 6,000,000 downloads +every month. You don't want to be left out! Testimonials ------------ Her Majesty's Government, Amazon, Google, Twilio, Runscope, Mozilla, Heroku, PayPal, NPR, Obama for America, Transifex, Native Instruments, The Washington -Post, Twitter, SoundCloud, Kippt, Readability, Sony, and Federal US Institutions that prefer to be unnamed -use Requests internally. It has been downloaded over 60,000,000 times from PyPI. +Post, Twitter, SoundCloud, Kippt, Readability, Sony, and Federal U.S. +Institutions that prefer to be unnamed claim to use Requests internally. **Armin Ronacher** Requests is the perfect example how beautiful an API can be with the @@ -79,12 +84,17 @@ Requests is ready for today's web. - Basic/Digest Authentication - Elegant Key/Value Cookies - Automatic Decompression +- Automatic Content Decoding - Unicode Response Bodies - Multipart File Uploads +- HTTP(S) Proxy Support - Connection Timeouts -- ``.netrc`` support -- Python 2.6—3.5 -- Thread-safe. +- Streaming Downloads +- ``.netrc`` Support +- Chunked Requests +- Thread-safety + +Requests supports Python 2.6 — 3.5, and runs great on PyPy. User Guide From 87f04e2a3233639e0d394a0add24772054bec1da Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 05:59:49 -0500 Subject: [PATCH 043/182] docs cleanup --- README.rst | 5 ----- docs/index.rst | 9 ++------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 2fe6a85f..12791f6b 100644 --- a/README.rst +++ b/README.rst @@ -36,11 +36,6 @@ URLs, or to form-encode your POST data. Keep-alive and HTTP connection pooling are 100% automatic, powered by `urllib3 `_, which is embedded within Requests. -But wait, there's more! Headers, cookies, json/form data, multipart files, and -url parameters can all be provided with standard Python dictionaries. -Responses feature RFC-compliant redirection history, unicode/bytes bodies, a -case-insensitive header dictionary, and more. - Besides, all the cool kids are doing it. Requests is one of the most downloaded Python packages of all time, pulling in over 6,000,000 downloads every month. You don't want to be left out! diff --git a/docs/index.rst b/docs/index.rst index 5dccd3f7..eb879e1b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -38,18 +38,13 @@ URLs, or to form-encode your POST data. Keep-alive and HTTP connection pooling are 100% automatic, powered by `urllib3 `_, which is embedded within Requests. -But wait, there's more! Headers, cookies, json/form data, multipart files, and -url parameters can all be provided with standard Python dictionaries. -Responses feature RFC-compliant redirection history, unicode/bytes bodies, a -case-insensitive header dictionary, and more. +Testimonials +------------ Besides, all the cool kids are doing it. Requests is one of the most downloaded Python packages of all time, pulling in over 6,000,000 downloads every month. You don't want to be left out! -Testimonials ------------- - Her Majesty's Government, Amazon, Google, Twilio, Runscope, Mozilla, Heroku, PayPal, NPR, Obama for America, Transifex, Native Instruments, The Washington Post, Twitter, SoundCloud, Kippt, Readability, Sony, and Federal U.S. From 20290e3f2c55037dfced04e91550b2a213dcfa65 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 06:01:50 -0500 Subject: [PATCH 044/182] docs fix --- docs/index.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index eb879e1b..6439f6c1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -41,10 +41,6 @@ which is embedded within Requests. Testimonials ------------ -Besides, all the cool kids are doing it. Requests is one of the most -downloaded Python packages of all time, pulling in over 6,000,000 downloads -every month. You don't want to be left out! - Her Majesty's Government, Amazon, Google, Twilio, Runscope, Mozilla, Heroku, PayPal, NPR, Obama for America, Transifex, Native Instruments, The Washington Post, Twitter, SoundCloud, Kippt, Readability, Sony, and Federal U.S. @@ -66,6 +62,8 @@ Institutions that prefer to be unnamed claim to use Requests internally. Python HTTP: When in doubt, or when not in doubt, use Requests. Beautiful, simple, Pythonic. +Requests is one of the most downloaded Python packages of all time, pulling in +over 6,000,000 downloads every month. All the cool kids are doing it! Feature Support --------------- From 7461371361a718e497bdfc63641d0916904e3f04 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 3 Feb 2016 06:11:36 -0500 Subject: [PATCH 045/182] behold the power of requests --- README.rst | 2 ++ docs/index.rst | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 12791f6b..afe0a124 100644 --- a/README.rst +++ b/README.rst @@ -14,6 +14,8 @@ consumption. including: security vulnerabilities, verbose code, reinventing the wheel, constantly reading documentation, depression, headaches, or even death. +Behold, the power of Requests: + .. code-block:: python >>> r = requests.get('https://api.github.com/user', auth=('user', 'pass')) diff --git a/docs/index.rst b/docs/index.rst index 6439f6c1..e8dc326c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,7 +15,7 @@ consumption. including: security vulnerabilities, verbose code, reinventing the wheel, constantly reading documentation, depression, headaches, or even death. -:: +Behold, the power of Requests:: >>> r = requests.get('https://api.github.com/user', auth=('user', 'pass')) >>> r.status_code From 1c5c6eeb7a82d3825b538b077fb42803c37359ab Mon Sep 17 00:00:00 2001 From: Dmitry Dygalo Date: Wed, 3 Feb 2016 21:19:20 +0100 Subject: [PATCH 046/182] Fixed test execution. --- setup.py | 1 - test_requests.py | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 88b8e087..5991f93e 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import os -import platform import re import sys diff --git a/test_requests.py b/test_requests.py index d7154510..0193f06f 100755 --- a/test_requests.py +++ b/test_requests.py @@ -606,11 +606,13 @@ class TestRequests: def test_hook_receives_request_arguments(self, httpbin): def hook(resp, **kwargs): - # FIXME. Not executed assert resp is not None assert kwargs != {} - requests.Request('GET', httpbin(), hooks={'response': hook}) + s = requests.Session() + r = requests.Request('GET', httpbin(), hooks={'response': hook}) + prep = s.prepare_request(r) + s.send(prep) def test_session_hooks_are_used_with_no_request_hooks(self, httpbin): hook = lambda x, *args, **kwargs: x From 18b26d20f7d0623e4b3df9fa6538e86069df3cdb Mon Sep 17 00:00:00 2001 From: Dmitry Dygalo Date: Fri, 5 Feb 2016 13:21:57 +0100 Subject: [PATCH 047/182] Added tests module. --- Makefile | 4 +- tests/__init__.py | 1 + tests/compat.py | 20 ++ tests/conftest.py | 23 ++ test_requests.py => tests/test_requests.py | 398 ++++----------------- tests/test_utils.py | 242 +++++++++++++ 6 files changed, 358 insertions(+), 330 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/compat.py create mode 100644 tests/conftest.py rename test_requests.py => tests/test_requests.py (79%) create mode 100644 tests/test_utils.py diff --git a/Makefile b/Makefile index dea33a3f..60b5e000 100644 --- a/Makefile +++ b/Makefile @@ -6,10 +6,10 @@ init: test: # This runs all of the tests. To run an individual test, run py.test with # the -k flag, like "py.test -k test_path_is_not_double_encoded" - py.test test_requests.py + py.test tests coverage: - py.test --verbose --cov-report term --cov=requests test_requests.py + py.test --verbose --cov-report term --cov=requests tests ci: init py.test --junitxml=junit.xml diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..57d631c3 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# coding: utf-8 diff --git a/tests/compat.py b/tests/compat.py new file mode 100644 index 00000000..a26bd9f4 --- /dev/null +++ b/tests/compat.py @@ -0,0 +1,20 @@ +# coding: utf-8 +from requests.compat import is_py3 + + +try: + import StringIO +except ImportError: + import io as StringIO + +try: + from cStringIO import StringIO as cStringIO +except ImportError: + cStringIO = None + +if is_py3: + def u(s): + return s +else: + def u(s): + return s.decode('unicode-escape') diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..af20e54d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,23 @@ +# coding: utf-8 +import pytest +from requests.compat import urljoin + + +def prepare_url(value): + # Issue #1483: Make sure the URL always has a trailing slash + httpbin_url = value.url.rstrip('/') + '/' + + def inner(*suffix): + return urljoin(httpbin_url, '/'.join(suffix)) + + return inner + + +@pytest.fixture +def httpbin(httpbin): + return prepare_url(httpbin) + + +@pytest.fixture +def httpbin_secure(httpbin_secure): + return prepare_url(httpbin_secure) diff --git a/test_requests.py b/tests/test_requests.py similarity index 79% rename from test_requests.py rename to tests/test_requests.py index 0193f06f..0d1ca049 100755 --- a/test_requests.py +++ b/tests/test_requests.py @@ -16,7 +16,7 @@ import pytest from requests.adapters import HTTPAdapter from requests.auth import HTTPDigestAuth, _basic_auth_str from requests.compat import ( - Morsel, cookielib, getproxies, str, urljoin, urlparse, is_py3, + Morsel, cookielib, getproxies, str, urlparse, builtin_str, OrderedDict) from requests.cookies import cookiejar_from_dict, morsel_to_cookie from requests.exceptions import ( @@ -28,45 +28,7 @@ from requests.sessions import SessionRedirectMixin from requests.models import urlencode from requests.hooks import default_hooks -try: - import StringIO -except ImportError: - import io as StringIO - -try: - from multiprocessing.pool import ThreadPool -except ImportError: - ThreadPool = None - -if is_py3: - def u(s): - return s -else: - def u(s): - return s.decode('unicode-escape') - - -@pytest.fixture -def httpbin(httpbin): - # Issue #1483: Make sure the URL always has a trailing slash - httpbin_url = httpbin.url.rstrip('/') + '/' - - def inner(*suffix): - return urljoin(httpbin_url, '/'.join(suffix)) - - return inner - - -@pytest.fixture -def httpsbin_url(httpbin_secure): - # Issue #1483: Make sure the URL always has a trailing slash - httpbin_url = httpbin_secure.url.rstrip('/') + '/' - - def inner(*suffix): - return urljoin(httpbin_url, '/'.join(suffix)) - - return inner - +from .compat import StringIO, u # Requests to this URL should always fail with a connection timeout (nothing # listening on that port) @@ -344,8 +306,8 @@ class TestRequests: r = s.get(url) assert r.status_code == 200 - @pytest.mark.parametrize('url, exception', - ( + @pytest.mark.parametrize( + 'url, exception', ( # Connecting to an unknown domain should raise a ConnectionError ('http://doesnotexist.google.com', ConnectionError), # Connecting to an invalid port should raise a ConnectionError @@ -536,8 +498,8 @@ class TestRequests: headers={str('Content-Type'): 'application/octet-stream'}, data='\xff') # compat.str is unicode. - def test_pyopenssl_redirect(self, httpsbin_url, httpbin_ca_bundle): - requests.get(httpsbin_url('status', '301'), verify=httpbin_ca_bundle) + def test_pyopenssl_redirect(self, httpbin_secure, httpbin_ca_bundle): + requests.get(httpbin_secure('status', '301'), verify=httpbin_ca_bundle) def test_urlencoded_get_query_multivalued_param(self, httpbin): @@ -578,13 +540,13 @@ class TestRequests: assert b'name="b\'stuff\'"' not in prep.body def test_unicode_method_name(self, httpbin): - files = {'file': open('test_requests.py', 'rb')} + files = {'file': open(__file__, 'rb')} r = requests.request( method=u('POST'), url=httpbin('post'), files=files) assert r.status_code == 200 def test_unicode_method_name_with_request_object(self, httpbin): - files = {'file': open('test_requests.py', 'rb')} + files = {'file': open(__file__, 'rb')} s = requests.Session() req = requests.Request(u("POST"), httpbin('post'), files=files) prep = s.prepare_request(req) @@ -851,26 +813,6 @@ class TestRequests: assert r.request.url == pr.request.url assert r.request.headers == pr.request.headers - def test_get_auth_from_url(self): - url = 'http://user:pass@complex.url.com/path?query=yes' - assert ('user', 'pass') == requests.utils.get_auth_from_url(url) - - def test_get_auth_from_url_encoded_spaces(self): - url = 'http://user:pass%20pass@complex.url.com/path?query=yes' - assert ('user', 'pass pass') == requests.utils.get_auth_from_url(url) - - def test_get_auth_from_url_not_encoded_spaces(self): - url = 'http://user:pass pass@complex.url.com/path?query=yes' - assert ('user', 'pass pass') == requests.utils.get_auth_from_url(url) - - def test_get_auth_from_url_percent_chars(self): - url = 'http://user%25user:pass@complex.url.com/path?query=yes' - assert ('user%user', 'pass') == requests.utils.get_auth_from_url(url) - - def test_get_auth_from_url_encoded_hashes(self): - url = 'http://user:pass%23pass@complex.url.com/path?query=yes' - assert ('user', 'pass#pass') == requests.utils.get_auth_from_url(url) - def test_cannot_send_unprepared_requests(self, httpbin): r = requests.Request(url=httpbin()) with pytest.raises(ValueError): @@ -986,23 +928,10 @@ class TestRequests: assert 'unicode' in p.headers.keys() assert 'byte' in p.headers.keys() - def test_can_send_nonstring_objects_with_files(self, httpbin): - data = {'a': 0.0} - files = {'b': 'foo'} - r = requests.Request('POST', httpbin('post'), data=data, files=files) - p = r.prepare() - - assert 'multipart/form-data' in p.headers['Content-Type'] - - def test_can_send_bytes_bytearray_objects_with_files(self, httpbin): - # Test bytes: + @pytest.mark.parametrize('files', ('foo', b'foo', bytearray(b'foo'))) + def test_can_send_objects_with_files(self, httpbin, files): data = {'a': 'this is a string'} - files = {'b': b'foo'} - r = requests.Request('POST', httpbin('post'), data=data, files=files) - p = r.prepare() - assert 'multipart/form-data' in p.headers['Content-Type'] - # Test bytearrays: - files = {'b': bytearray(b'foo')} + files = {'b': files} r = requests.Request('POST', httpbin('post'), data=data, files=files) p = r.prepare() assert 'multipart/form-data' in p.headers['Content-Type'] @@ -1145,43 +1074,10 @@ class TestRequests: assert len(list(r.iter_lines())) == 3 -class TestContentEncodingDetection: - - def test_none(self): - encodings = requests.utils.get_encodings_from_content('') - assert not len(encodings) - - @pytest.mark.parametrize( - 'content', ( - # HTML5 meta charset attribute - '', - # HTML4 pragma directive - '', - # XHTML 1.x served with text/html MIME type - '', - # XHTML 1.x served as XML - '', - ) - ) - def test_pragmas(self, content): - encodings = requests.utils.get_encodings_from_content(content) - assert len(encodings) == 1 - assert encodings[0] == 'UTF-8' - - def test_precedence(self): - content = ''' - - - - '''.strip() - encodings = requests.utils.get_encodings_from_content(content) - assert encodings == ['HTML5', 'HTML4', 'XML'] - - class TestCaseInsensitiveDict: - @pytest.mark.parametrize('cid', - ( + @pytest.mark.parametrize( + 'cid', ( CaseInsensitiveDict({'Foo': 'foo', 'BAr': 'bar'}), CaseInsensitiveDict([('Foo', 'foo'), ('BAr', 'bar')]), CaseInsensitiveDict(FOO='foo', BAr='bar'), @@ -1321,146 +1217,6 @@ class TestCaseInsensitiveDict: assert cid != cid_copy -class TestUtils: - - def test_super_len_io_streams(self): - """ Ensures that we properly deal with different kinds of IO streams. """ - # uses StringIO or io.StringIO (see import above) - from io import BytesIO - from requests.utils import super_len - - assert super_len(StringIO.StringIO()) == 0 - assert super_len( - StringIO.StringIO('with so much drama in the LBC')) == 29 - - assert super_len(BytesIO()) == 0 - assert super_len( - BytesIO(b"it's kinda hard bein' snoop d-o-double-g")) == 40 - - try: - import cStringIO - except ImportError: - pass - else: - assert super_len( - cStringIO.StringIO('but some how, some way...')) == 25 - - def test_super_len_correctly_calculates_len_of_partially_read_file(self): - """Ensure that we handle partially consumed file like objects.""" - from requests.utils import super_len - s = StringIO.StringIO() - s.write('foobarbogus') - assert super_len(s) == 0 - - def test_get_environ_proxies_ip_ranges(self): - """Ensures that IP addresses are correctly matches with ranges - in no_proxy variable.""" - from requests.utils import get_environ_proxies - os.environ['no_proxy'] = "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1" - assert get_environ_proxies('http://192.168.0.1:5000/') == {} - assert get_environ_proxies('http://192.168.0.1/') == {} - assert get_environ_proxies('http://172.16.1.1/') == {} - assert get_environ_proxies('http://172.16.1.1:5000/') == {} - assert get_environ_proxies('http://192.168.1.1:5000/') != {} - assert get_environ_proxies('http://192.168.1.1/') != {} - - def test_get_environ_proxies(self): - """Ensures that IP addresses are correctly matches with ranges - in no_proxy variable.""" - from requests.utils import get_environ_proxies - os.environ['no_proxy'] = "127.0.0.1,localhost.localdomain,192.168.0.0/24,172.16.1.1" - assert get_environ_proxies( - 'http://localhost.localdomain:5000/v1.0/') == {} - assert get_environ_proxies('http://www.requests.com/') != {} - - def test_select_proxies(self): - """Make sure we can select per-host proxies correctly.""" - from requests.utils import select_proxy - proxies = {'http': 'http://http.proxy', - 'http://some.host': 'http://some.host.proxy'} - assert select_proxy('hTTp://u:p@Some.Host/path', proxies) == 'http://some.host.proxy' - assert select_proxy('hTTp://u:p@Other.Host/path', proxies) == 'http://http.proxy' - assert select_proxy('hTTps://Other.Host', proxies) is None - - def test_guess_filename_when_int(self): - from requests.utils import guess_filename - assert None is guess_filename(1) - - def test_guess_filename_when_filename_is_an_int(self): - from requests.utils import guess_filename - fake = type('Fake', (object,), {'name': 1})() - assert None is guess_filename(fake) - - def test_guess_filename_with_file_like_obj(self): - from requests.utils import guess_filename - from requests import compat - fake = type('Fake', (object,), {'name': b'value'})() - guessed_name = guess_filename(fake) - assert b'value' == guessed_name - assert isinstance(guessed_name, compat.bytes) - - def test_guess_filename_with_unicode_name(self): - from requests.utils import guess_filename - from requests import compat - filename = b'value'.decode('utf-8') - fake = type('Fake', (object,), {'name': filename})() - guessed_name = guess_filename(fake) - assert filename == guessed_name - assert isinstance(guessed_name, compat.str) - - def test_is_ipv4_address(self): - from requests.utils import is_ipv4_address - assert is_ipv4_address('8.8.8.8') - assert not is_ipv4_address('8.8.8.8.8') - assert not is_ipv4_address('localhost.localdomain') - - def test_is_valid_cidr(self): - from requests.utils import is_valid_cidr - assert not is_valid_cidr('8.8.8.8') - assert is_valid_cidr('192.168.1.0/24') - - def test_dotted_netmask(self): - from requests.utils import dotted_netmask - assert dotted_netmask(8) == '255.0.0.0' - assert dotted_netmask(24) == '255.255.255.0' - assert dotted_netmask(25) == '255.255.255.128' - - def test_address_in_network(self): - from requests.utils import address_in_network - assert address_in_network('192.168.1.1', '192.168.1.0/24') - assert not address_in_network('172.16.0.1', '192.168.1.0/24') - - def test_get_auth_from_url(self): - """Ensures that username and password in well-encoded URI as per - RFC 3986 are correctly extracted.""" - from requests.utils import get_auth_from_url - from requests.compat import quote - percent_encoding_test_chars = "%!*'();:@&=+$,/?#[] " - url_address = "request.com/url.html#test" - url = "http://" + quote( - percent_encoding_test_chars, '') + ':' + quote( - percent_encoding_test_chars, '') + '@' + url_address - (username, password) = get_auth_from_url(url) - assert username == percent_encoding_test_chars - assert password == percent_encoding_test_chars - - def test_requote_uri_with_unquoted_percents(self): - """Ensure we handle unquoted percent signs in redirects. - - See: https://github.com/kennethreitz/requests/issues/2356 - """ - from requests.utils import requote_uri - bad_uri = 'http://example.com/fiz?buz=%ppicture' - quoted = 'http://example.com/fiz?buz=%25ppicture' - assert quoted == requote_uri(bad_uri) - - def test_requote_uri_properly_requotes(self): - """Ensure requoting doesn't break expectations.""" - from requests.utils import requote_uri - quoted = 'http://example.com/fiz?buz=%25ppicture' - assert quoted == requote_uri(quoted) - - class TestMorselToCookieExpires: """Tests for morsel_to_cookie when morsel contains expires.""" @@ -1472,8 +1228,8 @@ class TestMorselToCookieExpires: cookie = morsel_to_cookie(morsel) assert cookie.expires == 1 - @pytest.mark.parametrize('value, exception', - ( + @pytest.mark.parametrize( + 'value, exception', ( (100, TypeError), ('woops', ValueError), ) @@ -1516,20 +1272,23 @@ class TestMorselToCookieMaxAge: class TestTimeout: + def test_stream_timeout(self, httpbin): try: requests.get(httpbin('delay/10'), timeout=2.0) except requests.exceptions.Timeout as e: assert 'Read timed out' in e.args[0].args[0] - def test_invalid_timeout(self, httpbin): + @pytest.mark.parametrize( + 'timeout, error_text', ( + ((3, 4, 5), '(connect, read)'), + ('foo', 'must be an int or float'), + ) + ) + def test_invalid_timeout(self, httpbin, timeout, error_text): with pytest.raises(ValueError) as e: - requests.get(httpbin('get'), timeout=(3, 4, 5)) - assert '(connect, read)' in str(e) - - with pytest.raises(ValueError) as e: - requests.get(httpbin('get'), timeout="foo") - assert 'must be an int or float' in str(e) + requests.get(httpbin('get'), timeout=timeout) + assert error_text in str(e) def test_none_timeout(self, httpbin): """ Check that you can set None as a valid timeout value. @@ -1606,7 +1365,13 @@ class RedirectSession(SessionRedirectMixin): return string -class TestRedirects: +def test_requests_are_updated_each_time(httpbin): + session = RedirectSession([303, 307]) + prep = requests.Request('POST', httpbin('post')).prepare() + r0 = session.send(prep) + assert r0.request.method == 'POST' + assert session.calls[-1] == SendCall((r0.request,), {}) + redirect_generator = session.resolve_redirects(r0, prep) default_keyword_args = { 'stream': False, 'verify': True, @@ -1615,85 +1380,62 @@ class TestRedirects: 'allow_redirects': False, 'proxies': {}, } - - def test_requests_are_updated_each_time(self, httpbin): - session = RedirectSession([303, 307]) - prep = requests.Request('POST', httpbin('post')).prepare() - r0 = session.send(prep) - assert r0.request.method == 'POST' - assert session.calls[-1] == SendCall((r0.request,), {}) - redirect_generator = session.resolve_redirects(r0, prep) - for response in redirect_generator: - assert response.request.method == 'GET' - send_call = SendCall((response.request,), - TestRedirects.default_keyword_args) - assert session.calls[-1] == send_call + for response in redirect_generator: + assert response.request.method == 'GET' + send_call = SendCall((response.request,), default_keyword_args) + assert session.calls[-1] == send_call -@pytest.fixture -def list_of_tuples(): - return [ +@pytest.mark.parametrize( + 'data', ( (('a', 'b'), ('c', 'd')), (('c', 'd'), ('a', 'b')), (('a', 'b'), ('c', 'd'), ('e', 'f')), - ] - - -def test_data_argument_accepts_tuples(list_of_tuples): + ) +) +def test_data_argument_accepts_tuples(data): """Ensure that the data argument will accept tuples of strings and properly encode them. """ - for data in list_of_tuples: - p = PreparedRequest() - p.prepare( - method='GET', - url='http://www.example.com', - data=data, - hooks=default_hooks() - ) - assert p.body == urlencode(data) - - -def assert_copy(p, p_copy): - for attr in ('method', 'url', 'headers', '_cookies', 'body', 'hooks'): - assert getattr(p, attr) == getattr(p_copy, attr) - - -def test_prepared_request_empty_copy(): - p = PreparedRequest() - assert_copy(p, p.copy()) - - -def test_prepared_request_no_cookies_copy(): p = PreparedRequest() p.prepare( method='GET', url='http://www.example.com', - data='foo=bar', + data=data, hooks=default_hooks() ) - assert_copy(p, p.copy()) + assert p.body == urlencode(data) -def test_prepared_request_complete_copy(): - p = PreparedRequest() - p.prepare( - method='GET', - url='http://www.example.com', - data='foo=bar', - hooks=default_hooks(), - cookies={'foo': 'bar'} +@pytest.mark.parametrize( + 'kwargs', ( + None, + { + 'method': 'GET', + 'url': 'http://www.example.com', + 'data': 'foo=bar', + 'hooks': default_hooks() + }, + { + 'method': 'GET', + 'url': 'http://www.example.com', + 'data': 'foo=bar', + 'hooks': default_hooks(), + 'cookies': {'foo': 'bar'} + }, + { + 'method': 'GET', + 'url': u('http://www.example.com/üniçø∂é') + }, ) - assert_copy(p, p.copy()) - - -def test_prepare_unicode_url(): +) +def test_prepared_copy(kwargs): p = PreparedRequest() - p.prepare( - method='GET', - url=u('http://www.example.com/üniçø∂é'), - ) - assert_copy(p, p.copy()) + if kwargs: + p.prepare(**kwargs) + copy = p.copy() + for attr in ('method', 'url', 'headers', '_cookies', 'body', 'hooks'): + assert getattr(p, attr) == getattr(copy, attr) def test_urllib3_retries(httpbin): diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..0f7aec34 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,242 @@ +# coding: utf-8 +import os +from io import BytesIO + +import pytest +from requests import compat +from requests.utils import ( + address_in_network, dotted_netmask, + get_auth_from_url, get_encodings_from_content, + get_environ_proxies, guess_filename, + is_ipv4_address, is_valid_cidr, requote_uri, + select_proxy, super_len) + +from .compat import StringIO, cStringIO + + +class TestSuperLen: + + @pytest.mark.parametrize( + 'stream, value', ( + (StringIO.StringIO, 'Test'), + (BytesIO, b'Test'), + pytest.mark.skipif('cStringIO is None')( + (cStringIO, 'Test') + ), + )) + def test_io_streams(self, stream, value): + """ Ensures that we properly deal with different kinds of IO streams. """ + assert super_len(stream()) == 0 + assert super_len(stream(value)) == 4 + + def test_super_len_correctly_calculates_len_of_partially_read_file(self): + """Ensure that we handle partially consumed file like objects.""" + s = StringIO.StringIO() + s.write('foobarbogus') + assert super_len(s) == 0 + + +class TestGetEnvironProxies: + """Ensures that IP addresses are correctly matches with ranges + in no_proxy variable.""" + + @pytest.yield_fixture(scope='class', autouse=True, params=['no_proxy', 'NO_PROXY']) + def no_proxy(self, request): + os.environ[request.param] = '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1' + yield + del os.environ[request.param] + + @pytest.mark.parametrize( + 'url', ( + 'http://192.168.0.1:5000/', + 'http://192.168.0.1/', + 'http://172.16.1.1/', + 'http://172.16.1.1:5000/', + 'http://localhost.localdomain:5000/v1.0/', + ) + ) + def test_bypass(self, url): + assert get_environ_proxies(url) == {} + + @pytest.mark.parametrize( + 'url', ( + 'http://192.168.1.1:5000/', + 'http://192.168.1.1/', + 'http://www.requests.com/', + ) + ) + def test_not_bypass(self, url): + assert get_environ_proxies(url) != {} + + +class TestIsIPv4Address: + + def test_valid(self): + assert is_ipv4_address('8.8.8.8') + + @pytest.mark.parametrize('value', ('8.8.8.8.8', 'localhost.localdomain')) + def test_invalid(self, value): + assert not is_ipv4_address(value) + + +class TestIsValidCIDR: + + def test_valid(self): + assert is_valid_cidr('192.168.1.0/24') + + @pytest.mark.parametrize( + 'value', ( + '8.8.8.8', + '192.168.1.0/a', + '192.168.1.0/128', + '192.168.1.0/-1', + '192.168.1.999/24', + ) + ) + def test_invalid(self, value): + assert not is_valid_cidr(value) + + +class TestAddressInNetwork: + + def test_valid(self): + assert address_in_network('192.168.1.1', '192.168.1.0/24') + + def test_invalid(self): + assert not address_in_network('172.16.0.1', '192.168.1.0/24') + + +class TestGuessFilename: + + @pytest.mark.parametrize( + 'value', (1, type('Fake', (object,), {'name': 1})()), + ) + def test_guess_filename_invalid(self, value): + assert guess_filename(value) is None + + @pytest.mark.parametrize( + 'value, expected_type', ( + (b'value', compat.bytes), + (b'value'.decode('utf-8'), compat.str) + ) + ) + def test_guess_filename_valid(self, value, expected_type): + obj = type('Fake', (object,), {'name': value})() + result = guess_filename(obj) + assert result == value + assert isinstance(result, expected_type) + + +class TestContentEncodingDetection: + + def test_none(self): + encodings = get_encodings_from_content('') + assert not len(encodings) + + @pytest.mark.parametrize( + 'content', ( + # HTML5 meta charset attribute + '', + # HTML4 pragma directive + '', + # XHTML 1.x served with text/html MIME type + '', + # XHTML 1.x served as XML + '', + ) + ) + def test_pragmas(self, content): + encodings = get_encodings_from_content(content) + assert len(encodings) == 1 + assert encodings[0] == 'UTF-8' + + def test_precedence(self): + content = ''' + + + + '''.strip() + assert get_encodings_from_content(content) == ['HTML5', 'HTML4', 'XML'] + + +USER = PASSWORD = "%!*'();:@&=+$,/?#[] " +ENCODED_USER = compat.quote(USER, '') +ENCODED_PASSWORD = compat.quote(PASSWORD, '') + + +@pytest.mark.parametrize( + 'url, auth', ( + ( + 'http://' + ENCODED_USER + ':' + ENCODED_PASSWORD + '@' + + 'request.com/url.html#test', + (USER, PASSWORD) + ), + ( + 'http://user:pass@complex.url.com/path?query=yes', + ('user', 'pass') + ), + ( + 'http://user:pass%20pass@complex.url.com/path?query=yes', + ('user', 'pass pass') + ), + ( + 'http://user:pass pass@complex.url.com/path?query=yes', + ('user', 'pass pass') + ), + ( + 'http://user%25user:pass@complex.url.com/path?query=yes', + ('user%user', 'pass') + ), + ( + 'http://user:pass%23pass@complex.url.com/path?query=yes', + ('user', 'pass#pass') + ), + ) +) +def test_get_auth_from_url(url, auth): + assert get_auth_from_url(url) == auth + + +@pytest.mark.parametrize( + 'uri, expected', ( + ( + # Ensure requoting doesn't break expectations + 'http://example.com/fiz?buz=%25ppicture', + 'http://example.com/fiz?buz=%25ppicture', + ), + ( + # Ensure we handle unquoted percent signs in redirects + 'http://example.com/fiz?buz=%ppicture', + 'http://example.com/fiz?buz=%25ppicture', + ), + ) +) +def test_requote_uri_with_unquoted_percents(uri, expected): + """See: https://github.com/kennethreitz/requests/issues/2356 + """ + assert requote_uri(uri) == expected + + +@pytest.mark.parametrize( + 'mask, expected', ( + (8, '255.0.0.0'), + (24, '255.255.255.0'), + (25, '255.255.255.128'), + ) +) +def test_dotted_netmask(mask, expected): + assert dotted_netmask(mask) == expected + + +@pytest.mark.parametrize( + 'url, expected', ( + ('hTTp://u:p@Some.Host/path', 'http://some.host.proxy'), + ('hTTp://u:p@Other.Host/path', 'http://http.proxy'), + ('hTTps://Other.Host', None), + ) +) +def test_select_proxies(url, expected): + """Make sure we can select per-host proxies correctly.""" + proxies = {'http': 'http://http.proxy', + 'http://some.host': 'http://some.host.proxy'} + assert select_proxy(url, proxies) == expected From bbadf47a8fca97c4bf77e795369bc511e9483a40 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 5 Feb 2016 15:53:59 -0500 Subject: [PATCH 048/182] syntax changes --- tests/test_requests.py | 67 +++++++++++++++++------------------------- tests/test_utils.py | 35 ++++++++-------------- 2 files changed, 39 insertions(+), 63 deletions(-) diff --git a/tests/test_requests.py b/tests/test_requests.py index 0d1ca049..1ee379c9 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -32,7 +32,7 @@ from .compat import StringIO, u # Requests to this URL should always fail with a connection timeout (nothing # listening on that port) -TARPIT = "http://10.255.255.1" +TARPIT = 'http://10.255.255.1' class TestRequests: @@ -48,15 +48,14 @@ class TestRequests: requests.patch requests.post - @pytest.mark.parametrize('exception, url', - ( + @pytest.mark.parametrize( + 'exception, url', ( (MissingSchema, 'hiwpefhipowhefopw'), (InvalidSchema, 'localhost:3128'), (InvalidSchema, 'localhost.localdomain:3128/'), (InvalidSchema, '10.122.1.1:3128/'), (InvalidURL, 'http://'), - ) - ) + )) def test_invalid_url(self, exception, url): with pytest.raises(exception): requests.get(url) @@ -88,12 +87,11 @@ class TestRequests: assert request.path_url == '/get/test%20case' - @pytest.mark.parametrize('url, expected', - ( + @pytest.mark.parametrize( + 'url, expected', ( ('http://example.com/path#fragment', 'http://example.com/path?a=b#fragment'), ('http://example.com/path?key=value#fragment', 'http://example.com/path?key=value&a=b#fragment') - ) - ) + )) def test_params_are_added_before_fragment(self, url, expected): request = requests.Request('GET', url, params={"a": "b"}).prepare() assert request.url == expected @@ -276,9 +274,7 @@ class TestRequests: @pytest.mark.parametrize('key', ('User-agent', 'user-agent')) def test_user_agent_transfers(self, httpbin, key): - heads = { - key: 'Mozilla/5.0 (github.com/kennethreitz/requests)' - } + heads = {key: 'Mozilla/5.0 (github.com/kennethreitz/requests)'} r = requests.get(httpbin('user-agent'), headers=heads) assert heads[key] in r.text @@ -314,8 +310,7 @@ class TestRequests: ('http://localhost:1', ConnectionError), # Inputing a URL that cannot be parsed should raise an InvalidURL error ('http://fe80::5054:ff:fe5a:fc0', InvalidURL) - ) - ) + )) def test_errors(self, url, exception): with pytest.raises(exception): requests.get(url, timeout=1) @@ -480,15 +475,13 @@ class TestRequests: r.content.decode('ascii') @pytest.mark.parametrize( - 'url, params', - ( + 'url, params', ( ('/get', {'foo': 'føø'}), ('/get', {'føø': 'føø'}), ('/get', {'føø': 'føø'}), ('/get', {'foo': 'foo'}), ('ø', {'foo': 'foo'}), - ) - ) + )) def test_unicode_get(self, httpbin, url, params): requests.get(httpbin(url), params=params) @@ -520,8 +513,7 @@ class TestRequests: {'stuff': u('ëlïxr').encode('utf-8')}, {'stuff': 'elixr'}, {'stuff': 'elixr'.encode('utf-8')}, - ) - ) + )) def test_unicode_multipart_post(self, httpbin, data): r = requests.post(httpbin('post'), data=data, @@ -530,11 +522,10 @@ class TestRequests: def test_unicode_multipart_post_fieldnames(self, httpbin): filename = os.path.splitext(__file__)[0] + '.py' - r = requests.Request(method='POST', - url=httpbin('post'), - data={'stuff'.encode('utf-8'): 'elixr'}, - files={'file': ('test_requests.py', - open(filename, 'rb'))}) + r = requests.Request( + method='POST', url=httpbin('post'), + data={'stuff'.encode('utf-8'): 'elixr'}, + files={'file': ('test_requests.py', open(filename, 'rb'))}) prep = r.prepare() assert b'name="stuff"' in prep.body assert b'name="b\'stuff\'"' not in prep.body @@ -548,10 +539,10 @@ class TestRequests: def test_unicode_method_name_with_request_object(self, httpbin): files = {'file': open(__file__, 'rb')} s = requests.Session() - req = requests.Request(u("POST"), httpbin('post'), files=files) + req = requests.Request(u('POST'), httpbin('post'), files=files) prep = s.prepare_request(req) assert isinstance(prep.method, builtin_str) - assert prep.method == "POST" + assert prep.method == 'POST' resp = s.send(prep) assert resp.status_code == 200 @@ -560,9 +551,10 @@ class TestRequests: r = requests.post( httpbin('post'), data={'stuff': json.dumps({'a': 123})}, - files={'file1': ('test_requests.py', open(__file__, 'rb')), - 'file2': ('test_requests', open(__file__, 'rb'), - 'text/py-content-type')}) + files={ + 'file1': ('test_requests.py', open(__file__, 'rb')), + 'file2': ('test_requests', open(__file__, 'rb'), + 'text/py-content-type')}) assert r.status_code == 200 assert b"text/py-content-type" in r.request.body @@ -1081,8 +1073,7 @@ class TestCaseInsensitiveDict: CaseInsensitiveDict({'Foo': 'foo', 'BAr': 'bar'}), CaseInsensitiveDict([('Foo', 'foo'), ('BAr', 'bar')]), CaseInsensitiveDict(FOO='foo', BAr='bar'), - ) - ) + )) def test_init(self, cid): assert len(cid) == 2 assert 'foo' in cid @@ -1232,8 +1223,7 @@ class TestMorselToCookieExpires: 'value, exception', ( (100, TypeError), ('woops', ValueError), - ) - ) + )) def test_expires_invalid_int(self, value, exception): """Test case where an invalid type is passed for expires.""" morsel = Morsel() @@ -1283,8 +1273,7 @@ class TestTimeout: 'timeout, error_text', ( ((3, 4, 5), '(connect, read)'), ('foo', 'must be an int or float'), - ) - ) + )) def test_invalid_timeout(self, httpbin, timeout, error_text): with pytest.raises(ValueError) as e: requests.get(httpbin('get'), timeout=timeout) @@ -1391,8 +1380,7 @@ def test_requests_are_updated_each_time(httpbin): (('a', 'b'), ('c', 'd')), (('c', 'd'), ('a', 'b')), (('a', 'b'), ('c', 'd'), ('e', 'f')), - ) -) + )) def test_data_argument_accepts_tuples(data): """Ensure that the data argument will accept tuples of strings and properly encode them. @@ -1427,8 +1415,7 @@ def test_data_argument_accepts_tuples(data): 'method': 'GET', 'url': u('http://www.example.com/üniçø∂é') }, - ) -) + )) def test_prepared_copy(kwargs): p = PreparedRequest() if kwargs: diff --git a/tests/test_utils.py b/tests/test_utils.py index 0f7aec34..5a50e366 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -20,12 +20,10 @@ class TestSuperLen: 'stream, value', ( (StringIO.StringIO, 'Test'), (BytesIO, b'Test'), - pytest.mark.skipif('cStringIO is None')( - (cStringIO, 'Test') - ), + pytest.mark.skipif('cStringIO is None')((cStringIO, 'Test')), )) def test_io_streams(self, stream, value): - """ Ensures that we properly deal with different kinds of IO streams. """ + """Ensures that we properly deal with different kinds of IO streams.""" assert super_len(stream()) == 0 assert super_len(stream(value)) == 4 @@ -38,7 +36,7 @@ class TestSuperLen: class TestGetEnvironProxies: """Ensures that IP addresses are correctly matches with ranges - in no_proxy variable.""" + in no_proxy variable.""" @pytest.yield_fixture(scope='class', autouse=True, params=['no_proxy', 'NO_PROXY']) def no_proxy(self, request): @@ -53,8 +51,7 @@ class TestGetEnvironProxies: 'http://172.16.1.1/', 'http://172.16.1.1:5000/', 'http://localhost.localdomain:5000/v1.0/', - ) - ) + )) def test_bypass(self, url): assert get_environ_proxies(url) == {} @@ -63,8 +60,7 @@ class TestGetEnvironProxies: 'http://192.168.1.1:5000/', 'http://192.168.1.1/', 'http://www.requests.com/', - ) - ) + )) def test_not_bypass(self, url): assert get_environ_proxies(url) != {} @@ -91,8 +87,7 @@ class TestIsValidCIDR: '192.168.1.0/128', '192.168.1.0/-1', '192.168.1.999/24', - ) - ) + )) def test_invalid(self, value): assert not is_valid_cidr(value) @@ -118,8 +113,7 @@ class TestGuessFilename: 'value, expected_type', ( (b'value', compat.bytes), (b'value'.decode('utf-8'), compat.str) - ) - ) + )) def test_guess_filename_valid(self, value, expected_type): obj = type('Fake', (object,), {'name': value})() result = guess_filename(obj) @@ -143,8 +137,7 @@ class TestContentEncodingDetection: '', # XHTML 1.x served as XML '', - ) - ) + )) def test_pragmas(self, content): encodings = get_encodings_from_content(content) assert len(encodings) == 1 @@ -191,8 +184,7 @@ ENCODED_PASSWORD = compat.quote(PASSWORD, '') 'http://user:pass%23pass@complex.url.com/path?query=yes', ('user', 'pass#pass') ), - ) -) + )) def test_get_auth_from_url(url, auth): assert get_auth_from_url(url) == auth @@ -209,8 +201,7 @@ def test_get_auth_from_url(url, auth): 'http://example.com/fiz?buz=%ppicture', 'http://example.com/fiz?buz=%25ppicture', ), - ) -) + )) def test_requote_uri_with_unquoted_percents(uri, expected): """See: https://github.com/kennethreitz/requests/issues/2356 """ @@ -222,8 +213,7 @@ def test_requote_uri_with_unquoted_percents(uri, expected): (8, '255.0.0.0'), (24, '255.255.255.0'), (25, '255.255.255.128'), - ) -) + )) def test_dotted_netmask(mask, expected): assert dotted_netmask(mask) == expected @@ -233,8 +223,7 @@ def test_dotted_netmask(mask, expected): ('hTTp://u:p@Some.Host/path', 'http://some.host.proxy'), ('hTTp://u:p@Other.Host/path', 'http://http.proxy'), ('hTTps://Other.Host', None), - ) -) + )) def test_select_proxies(url, expected): """Make sure we can select per-host proxies correctly.""" proxies = {'http': 'http://http.proxy', From fc512d89245c4ee1f4b6d6f0445bc962a0621288 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 5 Feb 2016 17:51:09 -0500 Subject: [PATCH 049/182] Don't use redirect_cache if allow_redirects=False Fix for #2997 --- requests/sessions.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/requests/sessions.py b/requests/sessions.py index c177c602..639668f2 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -553,19 +553,21 @@ class Session(SessionRedirectMixin): if not isinstance(request, PreparedRequest): raise ValueError('You can only send PreparedRequests.') - checked_urls = set() - while request.url in self.redirect_cache: - checked_urls.add(request.url) - new_url = self.redirect_cache.get(request.url) - if new_url in checked_urls: - break - request.url = new_url - # Set up variables needed for resolve_redirects and dispatching of hooks allow_redirects = kwargs.pop('allow_redirects', True) stream = kwargs.get('stream') hooks = request.hooks + # Resolve URL in redirect cache, if available. + if allow_redirects: + checked_urls = set() + while request.url in self.redirect_cache: + checked_urls.add(request.url) + new_url = self.redirect_cache.get(request.url) + if new_url in checked_urls: + break + request.url = new_url + # Get the appropriate adapter to use adapter = self.get_adapter(url=request.url) From 087de5232c526f5a37dafdeeb7540ba79af44eed Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 5 Feb 2016 17:53:18 -0500 Subject: [PATCH 050/182] v2.9.2 in history --- HISTORY.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index f8c1a545..1fa26b7f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,13 @@ Release History --------------- +2.9.2 (???) ++++++++++++ + +**Bugfixes** + +- Don't use redirect_cache if allow_redirects=False + 2.9.1 (2015-12-21) ++++++++++++++++++ From 00c0a80e6bb51550bda18ab47e985ddd963b4303 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 5 Feb 2016 18:08:18 -0500 Subject: [PATCH 051/182] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index afe0a124..80f2edf7 100644 --- a/README.rst +++ b/README.rst @@ -39,7 +39,7 @@ are 100% automatic, powered by `urllib3 `_, which is embedded within Requests. Besides, all the cool kids are doing it. Requests is one of the most -downloaded Python packages of all time, pulling in over 6,000,000 downloads +downloaded Python packages of all time, pulling in over 7,000,000 downloads every month. You don't want to be left out! Feature Support From 3a77b59df222920391359f052cab9319dda31ac7 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 5 Feb 2016 18:08:48 -0500 Subject: [PATCH 052/182] Update index.rst --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index e8dc326c..0df56107 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -63,7 +63,7 @@ Institutions that prefer to be unnamed claim to use Requests internally. simple, Pythonic. Requests is one of the most downloaded Python packages of all time, pulling in -over 6,000,000 downloads every month. All the cool kids are doing it! +over 7,000,000 downloads every month. All the cool kids are doing it! Feature Support --------------- From 476938ab8d6722e76d0311cde8230f1ed524a01e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 7 Feb 2016 03:43:15 -0500 Subject: [PATCH 053/182] Delete setup.cfg --- setup.cfg | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 5e409001..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[wheel] -universal = 1 From 2fa5fa6ac5e008f918d8bd6e4a0c253f63780f55 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 7 Feb 2016 03:43:35 -0500 Subject: [PATCH 054/182] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 60b5e000..ff014fae 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ chardet: publish: python setup.py register python setup.py sdist upload - python setup.py bdist_wheel upload + python setup.py bdist_wheel upload --universal docs-init: From a3ded28c5728775d3509649bd9ee981efe8f5b0b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 7 Feb 2016 03:45:31 -0500 Subject: [PATCH 055/182] Update Makefile --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index ff014fae..be5fecf2 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,7 @@ publish: python setup.py register python setup.py sdist upload python setup.py bdist_wheel upload --universal + rm -fr dist .egg docs-init: From 283491f8b6a4204427f91a3327cbe6840f73e253 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 7 Feb 2016 10:58:30 -0500 Subject: [PATCH 056/182] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index be5fecf2..b31be6fb 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ chardet: publish: python setup.py register python setup.py sdist upload - python setup.py bdist_wheel upload --universal + python setup.py bdist_wheel --universal upload rm -fr dist .egg From 169b7edcc3c732dc949cfed377cbecda797b94ac Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 7 Feb 2016 11:51:23 -0500 Subject: [PATCH 057/182] Update README.rst --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 80f2edf7..9fe548d2 100644 --- a/README.rst +++ b/README.rst @@ -74,6 +74,7 @@ To install Requests, simply: .. code-block:: bash $ pip install requests + ✨🍰✨ Satisfaction, guaranteed. From 057722af23edf3f69bf7bdfed7c6c32cbe1ce2e7 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 10 Feb 2016 21:07:22 -0500 Subject: [PATCH 058/182] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b31be6fb..a8289712 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ publish: python setup.py register python setup.py sdist upload python setup.py bdist_wheel --universal upload - rm -fr dist .egg + rm -fr build dist .egg requests.egg-info docs-init: From b0704b86de4f207c11b72c1ea4046be95af5f9e5 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 18 Feb 2016 14:16:50 -0500 Subject: [PATCH 059/182] years --- requests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requests/__init__.py b/requests/__init__.py index bd5b5b97..1218d432 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -46,7 +46,7 @@ __version__ = '2.9.1' __build__ = 0x020901 __author__ = 'Kenneth Reitz' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2015 Kenneth Reitz' +__copyright__ = 'Copyright 2016 Kenneth Reitz' # Attempt to enable urllib3's SNI support, if possible try: From 9f33a0cd3c5106d4cc1b114f3ad57af45ab31daf Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 18 Feb 2016 20:40:46 -0500 Subject: [PATCH 060/182] refactor API documentation --- docs/api.rst | 109 ++++++++++++++++++++++++--------------------------- 1 file changed, 51 insertions(+), 58 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index b257f5ca..f3bc2107 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -25,9 +25,30 @@ They all return an instance of the :class:`Response ` object. .. autofunction:: patch .. autofunction:: delete +Exceptions +---------- + +.. autoexception:: requests.RequestException +.. autoexception:: requests.ConnectionError +.. autoexception:: requests.HTTPError +.. autoexception:: requests.URLRequired +.. autoexception:: requests.TooManyRedirects +.. autoexception:: requests.ConnectTimeout +.. autoexception:: requests.ReadTimeout +.. autoexception:: requests.Timeout + + +Request Sessions +---------------- + +.. _sessionapi: + +.. autoclass:: Session + :inherited-members: + Lower-Level Classes -~~~~~~~~~~~~~~~~~~~ +------------------- .. autoclass:: requests.Request :inherited-members: @@ -35,10 +56,11 @@ Lower-Level Classes .. autoclass:: Response :inherited-members: -Request Sessions ----------------- -.. autoclass:: Session +Lower-Lower-Level Classes +------------------------- + +.. autoclass:: requests.PreparedRequest :inherited-members: .. autoclass:: requests.adapters.HTTPAdapter @@ -52,21 +74,35 @@ Authentication .. autoclass:: requests.auth.HTTPProxyAuth .. autoclass:: requests.auth.HTTPDigestAuth -Exceptions -~~~~~~~~~~ -.. autoexception:: requests.exceptions.RequestException -.. autoexception:: requests.exceptions.ConnectionError -.. autoexception:: requests.exceptions.HTTPError -.. autoexception:: requests.exceptions.URLRequired -.. autoexception:: requests.exceptions.TooManyRedirects -.. autoexception:: requests.exceptions.ConnectTimeout -.. autoexception:: requests.exceptions.ReadTimeout -.. autoexception:: requests.exceptions.Timeout + +Encodings +--------- + +.. autofunction:: requests.utils.get_encodings_from_content +.. autofunction:: requests.utils.get_encoding_from_headers +.. autofunction:: requests.utils.get_unicode_from_response + + +.. _api-cookies: + +Cookies +------- + +.. autofunction:: requests.utils.dict_from_cookiejar +.. autofunction:: requests.utils.cookiejar_from_dict +.. autofunction:: requests.utils.add_dict_to_cookiejar + +.. autoclass:: requests.cookies.RequestsCookieJar + :inherited-members: + +.. autoclass:: requests.cookies.CookieConflictError + :inherited-members: + Status Code Lookup -~~~~~~~~~~~~~~~~~~ +------------------ .. autofunction:: requests.codes @@ -81,49 +117,6 @@ Status Code Lookup >>> requests.codes['\o/'] 200 -.. _api-cookies: - -Cookies -~~~~~~~ - -.. autofunction:: requests.utils.dict_from_cookiejar -.. autofunction:: requests.utils.cookiejar_from_dict -.. autofunction:: requests.utils.add_dict_to_cookiejar - -.. autoclass:: requests.cookies.RequestsCookieJar - :inherited-members: - -.. autoclass:: requests.cookies.CookieConflictError - :inherited-members: - - -Encodings -~~~~~~~~~ - -.. autofunction:: requests.utils.get_encodings_from_content -.. autofunction:: requests.utils.get_encoding_from_headers -.. autofunction:: requests.utils.get_unicode_from_response - - -Classes -~~~~~~~ - -.. autoclass:: requests.Response - :inherited-members: - -.. autoclass:: requests.Request - :inherited-members: - -.. autoclass:: requests.PreparedRequest - :inherited-members: - -.. _sessionapi: - -.. autoclass:: requests.Session - :inherited-members: - -.. autoclass:: requests.adapters.HTTPAdapter - :inherited-members: Migrating to 1.x From c1c393122d4975b166fa58fc5f62e4e789904e76 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 18 Feb 2016 20:40:59 -0500 Subject: [PATCH 061/182] change up marketing, update js/css --- docs/_templates/layout.html | 61 +++++++++---------------------- docs/_templates/sidebarintro.html | 39 ++++++++++---------- docs/_templates/sidebarlogo.html | 11 +++--- 3 files changed, 42 insertions(+), 69 deletions(-) diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index 3f435497..b53db050 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -11,6 +11,7 @@ {% endblock %} @@ -23,64 +24,36 @@ Fork me on GitHub - - - - - + (function() { + window._pa = window._pa || {}; + _pa.productId = "requests-docs"; + var pa = document.createElement('script'); pa.type = 'text/javascript'; pa.async = true; + pa.src = ('https:' == document.location.protocol ? 'https:' : 'http:') + "//tag.perfectaudience.com/serve/5226171f87bc6890da0000a0.js"; + var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(pa, s); + })(); + {%- endblock %} diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html index d2cbfe9d..38661e00 100644 --- a/docs/_templates/sidebarintro.html +++ b/docs/_templates/sidebarintro.html @@ -14,29 +14,15 @@ human beings.

- -

- Buy Requests Pro -

- - -

Get Updates

+

Stay Informed

Receive updates on new releases and upcoming projects.

-

Subscribe to Newsletter

+

+

-

Translations

- +

Join Mailing List.

Useful Links

+ + +

Translations

+ + + diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index 928cd2fd..5fcfae19 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -14,13 +14,12 @@ development release.

-

- Buy Requests Pro -

- -

Get Updates

+

Stay Informed

Receive updates on new releases and upcoming projects.

-

Subscribe to Newsletter

+

Join Mailing List.

+

+

\ No newline at end of file From f6252c2203dc3f9152056955266cce7050bc8e85 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 18 Feb 2016 20:59:01 -0500 Subject: [PATCH 062/182] improved installation instructions --- docs/user/install.rst | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/docs/user/install.rst b/docs/user/install.rst index 5f0ef9c4..b14ee310 100644 --- a/docs/user/install.rst +++ b/docs/user/install.rst @@ -7,20 +7,16 @@ This part of the documentation covers the installation of Requests. The first step to using any software package is getting it properly installed. -Distribute & Pip ----------------- +Pip Install +----------- -Installing Requests is simple with `pip `_, just run -this in your terminal:: +To install Requests, simply run this simple command in your terminal of choice:: $ pip install requests -or, with `easy_install `_:: - - $ easy_install requests - -But, you really `shouldn't do that `_. - +If you don't have `pip `_ installed (tisk tisk!), +`this Python installation guide `_ +can guide you through the process. Get the Code ------------ @@ -32,16 +28,12 @@ You can either clone the public repository:: $ git clone git://github.com/kennethreitz/requests.git -Download the `tarball `_:: +Or, download the `tarball `_:: $ curl -OL https://github.com/kennethreitz/requests/tarball/master + # optionally, zipball is also available (for Windows users). -Or, download the `zipball `_:: - - $ curl -OL https://github.com/kennethreitz/requests/zipball/master - - -Once you have a copy of the source, you can embed it in your Python package, -or install it into your site-packages easily:: +Once you have a copy of the source, you can embed it in your own Python +package, or install it into your site-packages easily:: $ python setup.py install From 47fa07dcd844663e7062713f510dd34fdf231b2d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 18 Feb 2016 23:57:30 -0500 Subject: [PATCH 063/182] update requirements workflow --- requirements-to-freeze.txt | 3 +++ requirements.txt | 20 ++++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) create mode 100644 requirements-to-freeze.txt diff --git a/requirements-to-freeze.txt b/requirements-to-freeze.txt new file mode 100644 index 00000000..7dc46a32 --- /dev/null +++ b/requirements-to-freeze.txt @@ -0,0 +1,3 @@ +pytest +pytest-cov +pytest-httpbin \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ad5da761..f41e8aa1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,14 @@ -py==1.4.30 -pytest==2.8.1 -pytest-cov==2.1.0 -pytest-httpbin==0.0.7 -httpbin==0.4.0 -wheel +coverage==4.0.3 +decorator==4.0.9 +Flask==0.10.1 +httpbin==0.4.1 +itsdangerous==0.24 +Jinja2==2.8 +MarkupSafe==0.23 +py==1.4.31 +pytest==2.8.7 +pytest-cov==2.2.1 +pytest-httpbin==0.2.0 +six==1.10.0 +Werkzeug==0.11.4 +wheel==0.29.0 From 45992769b2a4adcb4859da73417e8da92e4ae967 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Thu, 18 Feb 2016 23:58:47 -0500 Subject: [PATCH 064/182] +sphinx --- requirements-to-freeze.txt | 3 ++- requirements.txt | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/requirements-to-freeze.txt b/requirements-to-freeze.txt index 7dc46a32..e8b9e354 100644 --- a/requirements-to-freeze.txt +++ b/requirements-to-freeze.txt @@ -1,3 +1,4 @@ pytest pytest-cov -pytest-httpbin \ No newline at end of file +pytest-httpbin +sphinx \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index f41e8aa1..3d29de0c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,22 @@ +alabaster==0.7.7 +Babel==2.2.0 coverage==4.0.3 decorator==4.0.9 +docutils==0.12 Flask==0.10.1 httpbin==0.4.1 itsdangerous==0.24 Jinja2==2.8 MarkupSafe==0.23 py==1.4.31 +Pygments==2.1.1 pytest==2.8.7 pytest-cov==2.2.1 pytest-httpbin==0.2.0 +pytz==2015.7 six==1.10.0 +snowballstemmer==1.2.1 +Sphinx==1.3.5 +sphinx-rtd-theme==0.1.9 Werkzeug==0.11.4 wheel==0.29.0 From e98d71de135fceff1ce08fc52adb78f239e2c3c0 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 00:56:14 -0500 Subject: [PATCH 065/182] docs cleanup --- docs/Makefile | 71 ++++++- docs/make.bat | 453 ++++++++++++++++++++++++----------------- docs/requirements.txt | 2 - docs/user/advanced.rst | 5 +- 4 files changed, 333 insertions(+), 198 deletions(-) delete mode 100644 docs/requirements.txt diff --git a/docs/Makefile b/docs/Makefile index 946ba445..08a2acf6 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -7,6 +7,11 @@ SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter @@ -14,8 +19,7 @@ ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - +.PHONY: help help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @@ -25,53 +29,66 @@ help: @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" +.PHONY: clean clean: - -rm -rf $(BUILDDIR)/* + rm -rf $(BUILDDIR)/* +.PHONY: html html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." +.PHONY: dirhtml dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." +.PHONY: singlehtml singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." +.PHONY: pickle pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." +.PHONY: json json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." +.PHONY: htmlhelp htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." +.PHONY: qthelp qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @@ -81,6 +98,16 @@ qthelp: @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Requests.qhc" +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @@ -90,11 +117,13 @@ devhelp: @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Requests" @echo "# devhelp" +.PHONY: epub epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." +.PHONY: latex latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @@ -102,22 +131,33 @@ latex: @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." +.PHONY: latexpdf latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." - make -C $(BUILDDIR)/latex all-pdf + $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." +.PHONY: man man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." +.PHONY: texinfo texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @@ -125,29 +165,52 @@ texinfo: @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." +.PHONY: info info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." +.PHONY: gettext gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." +.PHONY: changes changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." +.PHONY: linkcheck linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." +.PHONY: doctest doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/make.bat b/docs/make.bat index 4441160c..9eaf9b88 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -1,190 +1,263 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Requests.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Requests.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -:end +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + echo. coverage to run coverage check of the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +REM Check if sphinx-build is available and fallback to Python version if any +%SPHINXBUILD% 1>NUL 2>NUL +if errorlevel 9009 goto sphinx_python +goto sphinx_ok + +:sphinx_python + +set SPHINXBUILD=python -m sphinx.__init__ +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +:sphinx_ok + + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Requests.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Requests.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "coverage" ( + %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage + if errorlevel 1 exit /b 1 + echo. + echo.Testing of coverage in the sources finished, look at the ^ +results in %BUILDDIR%/coverage/python.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index bf596036..00000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -alabaster==0.7.3 -Sphinx==1.1.3 diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 9c31950d..d3bf3cf4 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -356,11 +356,11 @@ POST Multiple Multipart-Encoded Files ------------------------------------- You can send multiple files in one request. For example, suppose you want to -upload image files to an HTML form with a multiple file field 'images': +upload image files to an HTML form with a multiple file field 'images':: -To do that, just set files to a list of tuples of (form_field_name, file_info): +To do that, just set files to a list of tuples of ``(form_field_name, file_info)``: >>> url = 'http://httpbin.org/post' >>> multiple_files = [('images', ('foo.png', open('foo.png', 'rb'), 'image/png')), @@ -519,6 +519,7 @@ You can also configure proxies by setting the environment variables $ export HTTP_PROXY="http://10.10.1.10:3128" $ export HTTPS_PROXY="http://10.10.1.10:1080" + $ python >>> import requests >>> requests.get("http://example.org") From dbefff68214b9a65d53a6f13fc1f1e49d6c7c91f Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 01:23:31 -0500 Subject: [PATCH 066/182] sphinx is now working properly on RTD (grrrR) --- docs/MANIFEST.in | 1 - docs/_templates/hacks.html | 23 ++++ docs/_templates/layout.html | 59 ---------- docs/conf.py | 215 ++++++++++++++++++++++++++++-------- 4 files changed, 192 insertions(+), 106 deletions(-) delete mode 100644 docs/MANIFEST.in create mode 100644 docs/_templates/hacks.html delete mode 100644 docs/_templates/layout.html diff --git a/docs/MANIFEST.in b/docs/MANIFEST.in deleted file mode 100644 index fb1021bf..00000000 --- a/docs/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include HISTORY.rst README.rst LICENSE \ No newline at end of file diff --git a/docs/_templates/hacks.html b/docs/_templates/hacks.html new file mode 100644 index 00000000..b18e73d0 --- /dev/null +++ b/docs/_templates/hacks.html @@ -0,0 +1,23 @@ + + + + + + + diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html deleted file mode 100644 index b53db050..00000000 --- a/docs/_templates/layout.html +++ /dev/null @@ -1,59 +0,0 @@ -{%- extends "basic/layout.html" %} -{%- block extrahead %} - - {{ super() }} - - {% if theme_touch_icon %} - - {% endif %} - - - - - -{% endblock %} -{%- block relbar2 %}{% endblock %} -{%- block footer %} -
- © Copyright {{ copyright }}. -
- - Fork me on GitHub - - - - - - - - -{%- endblock %} diff --git a/docs/conf.py b/docs/conf.py index a84469db..00e4261c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- # # Requests documentation build configuration file, created by -# sphinx-quickstart on Sun Feb 13 23:54:25 2011. +# sphinx-quickstart on Fri Feb 19 00:05:47 2016. # -# This file is execfile()d with the current directory set to its containing dir. +# This file is execfile()d with the current directory set to its +# containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. @@ -11,34 +12,43 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import sys +import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# Insert Requests' path into the system. sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.abspath('_themes')) + import requests from requests import __version__ -import alabaster -# -- General configuration ----------------------------------------------------- +# -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx', - 'alabaster' + 'sphinx.ext.todo', + 'sphinx.ext.viewcode', ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] -# The suffix of source filenames. +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The encoding of source files. @@ -50,6 +60,7 @@ master_doc = 'index' # General information about the project. project = u'Requests' copyright = u'2016. A Kenneth Reitz Project' +author = u'Kenneth Reitz' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -58,11 +69,14 @@ copyright = u'2016. A tag referring to it. The value of this option must be the @@ -181,23 +208,45 @@ html_show_sphinx = False # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + # Output file base name for HTML help builder. htmlhelp_basename = 'Requestsdoc' +# -- Options for LaTeX output --------------------------------------------- -# -- Options for LaTeX output -------------------------------------------------- - -# The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} # Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'Requests.tex', u'Requests Documentation', - u'Kenneth Reitz', 'manual'), + (master_doc, 'Requests.tex', u'Requests Documentation', + u'Kenneth Reitz', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -214,9 +263,6 @@ latex_documents = [ # If true, show URL addresses after external links. #latex_show_urls = False -# Additional stuff for the LaTeX preamble. -#latex_preamble = '' - # Documents to append as an appendix to all manuals. #latex_appendices = [] @@ -224,33 +270,110 @@ latex_documents = [ #latex_domain_indices = True -# -- Options for manual page output -------------------------------------------- +# -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'requests', u'Requests Documentation', - [u'Kenneth Reitz'], 1) + (master_doc, 'requests', u'Requests Documentation', + [author], 1) ] # If true, show URL addresses after external links. #man_show_urls = False -# -- Options for Texinfo output ------------------------------------------------ + +# -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'Requests', u'Requests Documentation', u'Kenneth Reitz', - 'Requests', 'One line description of project.', 'Miscellaneous'), + (master_doc, 'Requests', u'Requests Documentation', + author, 'Requests', 'One line description of project.', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. -texinfo_appendices = [] +#texinfo_appendices = [] -sys.path.append(os.path.abspath('_themes')) -html_theme_path =[alabaster.get_path()] -html_theme = 'alabaster' +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False + + +# -- Options for Epub output ---------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project +epub_author = author +epub_publisher = author +epub_copyright = copyright + +# The basename for the epub file. It defaults to the project name. +#epub_basename = project + +# The HTML theme for the epub output. Since the default themes are not +# optimized for small screen space, using the same theme for HTML and epub +# output is usually not wise. This defaults to 'epub', a theme designed to save +# visual space. +#epub_theme = 'epub' + +# The language of the text. It defaults to the language option +# or 'en' if the language is not set. +#epub_language = '' + +# The scheme of the identifier. Typical schemes are ISBN or URL. +#epub_scheme = '' + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +#epub_identifier = '' + +# A unique identification for the text. +#epub_uid = '' + +# A tuple containing the cover image and cover page html template filenames. +#epub_cover = () + +# A sequence of (type, uri, title) tuples for the guide element of content.opf. +#epub_guide = () + +# HTML files that should be inserted before the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_pre_files = [] + +# HTML files that should be inserted after the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_post_files = [] + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + +# The depth of the table of contents in toc.ncx. +#epub_tocdepth = 3 + +# Allow duplicate toc entries. +#epub_tocdup = True + +# Choose between 'default' and 'includehidden'. +#epub_tocscope = 'default' + +# Fix unsupported image types using the Pillow. +#epub_fix_images = False + +# Scale large images. +#epub_max_image_width = 0 + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#epub_show_urls = 'inline' + +# If false, no index is generated. +#epub_use_index = True intersphinx_mapping = {'urllib3': ('http://urllib3.readthedocs.org/en/latest', None)} From 60b591eaac1836998550f555cd6d28a2c87b4f84 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 01:47:54 -0500 Subject: [PATCH 067/182] docs: widen things up --- docs/_templates/hacks.html | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/_templates/hacks.html b/docs/_templates/hacks.html index b18e73d0..997f8b23 100644 --- a/docs/_templates/hacks.html +++ b/docs/_templates/hacks.html @@ -2,8 +2,21 @@ @@ -21,3 +34,7 @@ s.parentNode.insertBefore(t, s); })(); + + + + \ No newline at end of file From f422ace07e92d79308f2e72897126971c37a3414 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 01:52:59 -0500 Subject: [PATCH 068/182] clean up codeblock styles --- docs/_templates/hacks.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/_templates/hacks.html b/docs/_templates/hacks.html index 997f8b23..4763b26b 100644 --- a/docs/_templates/hacks.html +++ b/docs/_templates/hacks.html @@ -12,11 +12,15 @@ /* Make the document a little wider, less code is cut-off. */ div.document {width: 1008px;} + /* Much-improved spacing around code blocks. */ + div.highlight pre {padding: 11px 14px;} + /* Remain Responsive! */ @media screen and (max-width: 1008px) { div.sphinxsidebar {display: none;} div.document {width: 100%!important;} } + From f4f863f168d6e2c660cd73d158aaf330ba9070da Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 02:06:33 -0500 Subject: [PATCH 069/182] cleanup advanced.rst single-quoted strings, yo! --- docs/user/advanced.rst | 47 ++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index d3bf3cf4..79ba227b 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -24,7 +24,7 @@ Let's persist some cookies across requests:: s = requests.Session() s.get('http://httpbin.org/cookies/set/sessioncookie/123456789') - r = s.get("http://httpbin.org/cookies") + r = s.get('http://httpbin.org/cookies') print(r.text) # '{"cookies": {"sessioncookie": "123456789"}}' @@ -50,6 +50,7 @@ requests, even if using a session. This example will only send the cookies with the first request, but not the second:: s = requests.Session() + r = s.get('http://httpbin.org/cookies', cookies={'from-my': 'browser'}) print(r.text) # '{"cookies": {"from-my": "browser"}}' @@ -129,14 +130,15 @@ request. The simple recipe for this is the following:: from requests import Request, Session s = Session() - req = Request('GET', url, - data=data, - headers=header - ) + + req = Request('POST', url, data=data, headers=headers) prepped = req.prepare() # do something with prepped.body + prepped.body = 'No, I want exactly this as the body.' + # do something with prepped.headers + del prepped.headers['Content-Type'] resp = s.send(prepped, stream=stream, @@ -491,7 +493,9 @@ set ``stream`` to ``True`` and iterate over the response with lines = r.iter_lines() # Save the first line for later or just skip it + first_line = next(lines) + for line in lines: print(line) @@ -506,11 +510,11 @@ If you need to use a proxy, you can configure individual requests with the import requests proxies = { - "http": "http://10.10.1.10:3128", - "https": "http://10.10.1.10:1080", + 'http': 'http://10.10.1.10:3128', + 'https': 'http://10.10.1.10:1080', } - requests.get("http://example.org", proxies=proxies) + requests.get('http://example.org', proxies=proxies) You can also configure proxies by setting the environment variables ``HTTP_PROXY`` and ``HTTPS_PROXY``. @@ -522,12 +526,12 @@ You can also configure proxies by setting the environment variables $ python >>> import requests - >>> requests.get("http://example.org") + >>> requests.get('http://example.org') To use HTTP Basic Auth with your proxy, use the `http://user:password@host/` syntax:: proxies = { - "http": "http://user:pass@10.10.1.10:3128/", + 'http': 'http://user:pass@10.10.1.10:3128/', } To give a proxy for a specific scheme and host, use the @@ -537,7 +541,7 @@ any request to the given scheme and exact hostname. :: proxies = { - "http://10.20.1.128": "http://10.10.1.10:5323", + 'http://10.20.1.128': 'http://10.10.1.10:5323', } Note that proxy URLs must include the scheme. @@ -603,10 +607,13 @@ So, GitHub returns JSON. That's great, we can use the :meth:`r.json :: >>> commit_data = r.json() + >>> print(commit_data.keys()) [u'committer', u'author', u'url', u'tree', u'sha', u'parents', u'message'] + >>> print(commit_data[u'committer']) {u'date': u'2012-05-10T11:10:50-07:00', u'email': u'me@kennethreitz.com', u'name': u'Kenneth Reitz'} + >>> print(commit_data[u'message']) makin' history @@ -646,9 +653,12 @@ already exists, we will use it as an example. Let's start by getting it. >>> r = requests.get('https://api.github.com/repos/kennethreitz/requests/issues/482') >>> r.status_code 200 + >>> issue = json.loads(r.text) + >>> print(issue[u'title']) Feature any http verb in docs + >>> print(issue[u'comments']) 3 @@ -659,9 +669,12 @@ Cool, we have three comments. Let's take a look at the last of them. >>> r = requests.get(r.url + u'/comments') >>> r.status_code 200 + >>> comments = r.json() + >>> print(comments[0].keys()) [u'body', u'url', u'created_at', u'updated_at', u'user', u'id'] + >>> print(comments[2][u'body']) Probably in the "advanced" section @@ -681,6 +694,7 @@ is to POST to the thread. Let's do it. >>> body = json.dumps({u"body": u"Sounds great! I'll get right on it!"}) >>> url = u"https://api.github.com/repos/kennethreitz/requests/issues/482/comments" + >>> r = requests.post(url=url, data=body) >>> r.status_code 404 @@ -693,9 +707,11 @@ the very common Basic Auth. >>> from requests.auth import HTTPBasicAuth >>> auth = HTTPBasicAuth('fake@example.com', 'not_a_real_password') + >>> r = requests.post(url=url, data=body, auth=auth) >>> r.status_code 201 + >>> content = r.json() >>> print(content[u'body']) Sounds great! I'll get right on it. @@ -709,8 +725,10 @@ that. >>> print(content[u"id"]) 5804413 + >>> body = json.dumps({u"body": u"Sounds great! I'll get right on it once I feed my cat."}) >>> url = u"https://api.github.com/repos/kennethreitz/requests/issues/comments/5804413" + >>> r = requests.patch(url=url, data=body, auth=auth) >>> r.status_code 200 @@ -831,10 +849,9 @@ SSLv3: """"Transport adapter" that allows us to use SSLv3.""" def init_poolmanager(self, connections, maxsize, block=False): - self.poolmanager = PoolManager(num_pools=connections, - maxsize=maxsize, - block=block, - ssl_version=ssl.PROTOCOL_SSLv3) + self.poolmanager = PoolManager( + num_pools=connections, maxsize=maxsize, + block=block, ssl_version=ssl.PROTOCOL_SSLv3) .. _`described here`: http://www.kennethreitz.org/essays/the-future-of-python-http .. _`urllib3`: https://github.com/shazow/urllib3 From bf8c8312ddece928b2549104b5864600df368904 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 02:12:15 -0500 Subject: [PATCH 070/182] cleanup quickstart --- docs/user/quickstart.rst | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 10037fe6..d31f224e 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -37,15 +37,15 @@ get all the information we need from this object. Requests' simple API means that all forms of HTTP request are as obvious. For example, this is how you make an HTTP POST request:: - >>> r = requests.post("http://httpbin.org/post", data = {"key":"value"}) + >>> r = requests.post('http://httpbin.org/post', data = {'key':'value'}) Nice, right? What about the other HTTP request types: PUT, DELETE, HEAD and OPTIONS? These are all just as simple:: - >>> r = requests.put("http://httpbin.org/put", data = {"key":"value"}) - >>> r = requests.delete("http://httpbin.org/delete") - >>> r = requests.head("http://httpbin.org/get") - >>> r = requests.options("http://httpbin.org/get") + >>> r = requests.put('http://httpbin.org/put', data = {'key':'value'}) + >>> r = requests.delete('http://httpbin.org/delete') + >>> r = requests.head('http://httpbin.org/get') + >>> r = requests.options('http://httpbin.org/get') That's all well and good, but it's also only the start of what Requests can do. @@ -63,7 +63,7 @@ Requests allows you to provide these arguments as a dictionary, using the following code:: >>> payload = {'key1': 'value1', 'key2': 'value2'} - >>> r = requests.get("http://httpbin.org/get", params=payload) + >>> r = requests.get('http://httpbin.org/get', params=payload) You can see that the URL has been correctly encoded by printing the URL:: @@ -76,7 +76,8 @@ URL's query string. You can also pass a list of items as a value:: >>> payload = {'key1': 'value1', 'key2': ['value2', 'value3']} - >>> r = requests.get("http://httpbin.org/get", params=payload) + + >>> r = requests.get('http://httpbin.org/get', params=payload) >>> print(r.url) http://httpbin.org/get?key1=value1&key2=value2&key2=value3 @@ -87,6 +88,7 @@ We can read the content of the server's response. Consider the GitHub timeline again:: >>> import requests + >>> r = requests.get('https://api.github.com/events') >>> r.text u'[{"repository":{"open_issues":0,"url":"https://github.com/... @@ -131,6 +133,7 @@ use the following code:: >>> from PIL import Image >>> from StringIO import StringIO + >>> i = Image.open(StringIO(r.content)) @@ -140,6 +143,7 @@ JSON Response Content There's also a builtin JSON decoder, in case you're dealing with JSON data:: >>> import requests + >>> r = requests.get('https://api.github.com/events') >>> r.json() [{u'repository': {u'open_issues': 0, u'url': 'https://github.com/... @@ -163,8 +167,10 @@ server, you can access ``r.raw``. If you want to do this, make sure you set ``stream=True`` in your initial request. Once you do, you can do this:: >>> r = requests.get('https://api.github.com/events', stream=True) + >>> r.raw + >>> r.raw.read(10) '\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03' @@ -189,7 +195,6 @@ If you'd like to add HTTP headers to a request, simply pass in a ``dict`` to the For example, we didn't specify our user-agent in the previous example:: - >>> import json >>> url = 'https://api.github.com/some/endpoint' >>> headers = {'user-agent': 'my-app/0.0.1'} @@ -215,6 +220,7 @@ To do this, simply pass a dictionary to the ``data`` argument. Your dictionary of data will automatically be form-encoded when the request is made:: >>> payload = {'key1': 'value1', 'key2': 'value2'} + >>> r = requests.post("http://httpbin.org/post", data=payload) >>> print(r.text) { @@ -232,6 +238,7 @@ you pass in a ``string`` instead of a ``dict``, that data will be posted directl For example, the GitHub API v3 accepts JSON-Encoded POST/PATCH data:: >>> import json + >>> url = 'https://api.github.com/some/endpoint' >>> payload = {'some': 'data'} @@ -426,10 +433,13 @@ response. For example, GitHub redirects all HTTP requests to HTTPS:: >>> r = requests.get('http://github.com') + >>> r.url 'https://github.com/' + >>> r.status_code 200 + >>> r.history [] @@ -438,16 +448,20 @@ If you're using GET, OPTIONS, POST, PUT, PATCH or DELETE, you can disable redirection handling with the ``allow_redirects`` parameter:: >>> r = requests.get('http://github.com', allow_redirects=False) + >>> r.status_code 301 + >>> r.history [] If you're using HEAD, you can enable redirection as well:: >>> r = requests.head('http://github.com', allow_redirects=True) + >>> r.url 'https://github.com/' + >>> r.history [] From f4445b3d4f320f66142cf5aa0656fe62a6c0df3e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 02:40:14 -0500 Subject: [PATCH 071/182] improve advanced.rst --- docs/user/advanced.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 79ba227b..cc99b679 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -167,15 +167,15 @@ applied, replace the call to :meth:`Request.prepare() from requests import Request, Session s = Session() - req = Request('GET', url, - data=data - headers=headers - ) + req = Request('GET', url, data=data, headers=headers) prepped = s.prepare_request(req) # do something with prepped.body + prepped.body = 'Seriously, send exactly these bytes.' + # do something with prepped.headers + prepped.headers['Keep-Dead'] = 'parrot' resp = s.send(prepped, stream=stream, @@ -362,11 +362,12 @@ upload image files to an HTML form with a multiple file field 'images':: -To do that, just set files to a list of tuples of ``(form_field_name, file_info)``: +To do that, just set files to a list of tuples of ``(form_field_name, file_info)``:: >>> url = 'http://httpbin.org/post' - >>> multiple_files = [('images', ('foo.png', open('foo.png', 'rb'), 'image/png')), - ('images', ('bar.png', open('bar.png', 'rb'), 'image/png'))] + >>> multiple_files = [ + ('images', ('foo.png', open('foo.png', 'rb'), 'image/png')), + ('images', ('bar.png', open('bar.png', 'rb'), 'image/png'))] >>> r = requests.post(url, files=multiple_files) >>> r.text { From f8af499700b58cf550205c463ad4b163a784892b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 02:45:07 -0500 Subject: [PATCH 072/182] improve further advanced.rst --- docs/user/advanced.rst | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index cc99b679..ddd6edf6 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -531,9 +531,7 @@ You can also configure proxies by setting the environment variables To use HTTP Basic Auth with your proxy, use the `http://user:password@host/` syntax:: - proxies = { - 'http': 'http://user:pass@10.10.1.10:3128/', - } + proxies = {'http': 'http://user:pass@10.10.1.10:3128/'} To give a proxy for a specific scheme and host, use the `scheme://hostname` form for the key. This will match for @@ -541,9 +539,7 @@ any request to the given scheme and exact hostname. :: - proxies = { - 'http://10.20.1.128': 'http://10.10.1.10:5323', - } + proxies = {'http://10.20.1.128': 'http://10.10.1.10:5323'} Note that proxy URLs must include the scheme. From f63106005b3050b5a99833ec47b78667dc4766b2 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 02:59:05 -0500 Subject: [PATCH 073/182] better codeblocks on mobile --- docs/_templates/hacks.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/_templates/hacks.html b/docs/_templates/hacks.html index 4763b26b..ba032452 100644 --- a/docs/_templates/hacks.html +++ b/docs/_templates/hacks.html @@ -19,6 +19,9 @@ @media screen and (max-width: 1008px) { div.sphinxsidebar {display: none;} div.document {width: 100%!important;} + + /* Have code blocks escape the document right-margin. */ + div.highlight pre {margin-right: -30px;} } From 688a00c38643f55df41f8337b063ba1944f33ca6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 03:47:12 -0500 Subject: [PATCH 074/182] improve contributions section --- docs/dev/contributing.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 075d042a..f619e569 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -22,12 +22,8 @@ contributors. .. _Ian Cordasco: http://www.coglib.com/~icordasc/ .. _Cory Benfield: https://lukasa.co.uk/about - -All Contributions ------------------ - Be Cordial -~~~~~~~~~~ +---------- **Be cordial or be on your way.** @@ -41,7 +37,7 @@ everyone involved is treated with respect. .. _early-feedback: Get Early Feedback -~~~~~~~~~~~~~~~~~~ +------------------ If you are contributing, do not feel the need to sit on your contribution until it is perfectly polished and complete. It helps everyone involved for you to @@ -51,7 +47,7 @@ getting that contribution accepted, and can save you from putting a lot of work into a contribution that is not suitable for the project. Contribution Suitability -~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------ The project maintainer has the last word on whether or not a contribution is suitable for Requests. All contributions will be considered, but from time @@ -116,6 +112,9 @@ When contributing documentation, please attempt to follow the style of the documentation files. This means a soft-limit of 79 characters wide in your text files and a semi-formal prose style. +When presenting Python code, use single-quoted strings (``'hello'`` instead of +``"hello"``). + .. _reStructuredText: http://docutils.sourceforge.net/rst.html .. _Sphinx: http://sphinx-doc.org/index.html From d3bd9f30a6acec62cafee067e82f7cb1cd775701 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 04:09:03 -0500 Subject: [PATCH 075/182] contributing --- docs/dev/contributing.rst | 45 +++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index f619e569..7aa763cd 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -3,33 +3,38 @@ Contributor's Guide =================== -If you're reading this you're probably interested in contributing to -Requests. First, We'd like to say: thank you! Open source projects -live-and-die based on the support they receive from others, and the fact that -you're even considering supporting Requests is very generous of -you. +If you're reading this, you're probably interested in contributing to Requests. +Thank you very much! Open source projects live-and-die based on the support +they receive from others, and the fact that you're even considering +contributing to the Requests project is *very* generous of you. -This document lays out guidelines and advice for contributing to Requests. -If you're thinking of contributing, start by reading this thoroughly and -getting a feel for how contributing to the project works. If you have any +This document lays out guidelines and advice for contributing to this project. +If you're thinking of contributing, please start by reading this document and +getting a feel for how contributing to this project works. If you have any questions, feel free to reach out to either `Ian Cordasco`_ or `Cory Benfield`_, the primary maintainers. +.. _Ian Cordasco: http://www.coglib.com/~icordasc/ +.. _Cory Benfield: https://lukasa.co.uk/about + +If you have non-technical feedback, philisophical ponderings, crazy ideas, or +other general thoughts about Requests or its position within the Python +ecosystem, the BDFL, `Kenneth Reitz`_, would love to hear from you. + The guide is split into sections based on the type of contribution you're thinking of making, with a section that covers general guidelines for all contributors. -.. _Ian Cordasco: http://www.coglib.com/~icordasc/ -.. _Cory Benfield: https://lukasa.co.uk/about +.. _Kenneth Reitz: mailto:me@kennethreitz.org Be Cordial ---------- -**Be cordial or be on your way.** + **Be cordial or be on your way**. *—Kenneth Reitz* Requests has one very important rule governing all forms of contribution, including reporting bugs or requesting features. This golden rule is -`be cordial or be on your way`_. **All contributions are welcome**, as long as +"`be cordial or be on your way`_". **All contributions are welcome**, as long as everyone involved is treated with respect. .. _be cordial or be on your way: http://kennethreitz.org/be-cordial-or-be-on-your-way/ @@ -108,9 +113,9 @@ the ``docs/`` directory of the codebase. They're written in `reStructuredText`_, and use `Sphinx`_ to generate the full suite of documentation. -When contributing documentation, please attempt to follow the style of the +When contributing documentation, please do your best to follow the style of the documentation files. This means a soft-limit of 79 characters wide in your text -files and a semi-formal prose style. +files and a semi-formal, yet friendly and approachable, prose style. When presenting Python code, use single-quoted strings (``'hello'`` instead of ``"hello"``). @@ -135,10 +140,14 @@ of other contributors, and should be avoided as much as possible. Feature Requests ---------------- -Requests is in a perpetual feature freeze. The maintainers believe that -requests contains every major feature currently required by the vast majority -of users. +Requests is in a perpetual feature freeze, only the BDFL can add or approve of +new features. The maintainers believe that Requests is a feature-complete +peice of software at this time. + +One of the most important skills to have while maintaining a largely-used +open source project is learning the ability to say "no" to suggested changes, +while keeping an open ear and mind. If you believe there is a feature missing, feel free to raise a feature request, but please do be aware that the overwhelming likelihood is that your -feature request will not be accepted. +feature request will not be accepted. \ No newline at end of file From 886f92324a4a030676fca440d33f34631c27acd9 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 04:10:37 -0500 Subject: [PATCH 076/182] contributing --- docs/dev/contributing.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 7aa763cd..ceedda33 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -34,7 +34,9 @@ Be Cordial Requests has one very important rule governing all forms of contribution, including reporting bugs or requesting features. This golden rule is -"`be cordial or be on your way`_". **All contributions are welcome**, as long as +"`be cordial or be on your way`_". + +**All contributions are welcome**, as long as everyone involved is treated with respect. .. _be cordial or be on your way: http://kennethreitz.org/be-cordial-or-be-on-your-way/ From 2cb052aa950a20744778af802acff240830505f3 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 04:11:41 -0500 Subject: [PATCH 077/182] sp --- docs/dev/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index ceedda33..094b7b19 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -17,7 +17,7 @@ the primary maintainers. .. _Ian Cordasco: http://www.coglib.com/~icordasc/ .. _Cory Benfield: https://lukasa.co.uk/about -If you have non-technical feedback, philisophical ponderings, crazy ideas, or +If you have non-technical feedback, philosophical ponderings, crazy ideas, or other general thoughts about Requests or its position within the Python ecosystem, the BDFL, `Kenneth Reitz`_, would love to hear from you. From 5d0ffd85cef1f85124ab38a9207f9d4d734910cf Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 04:30:06 -0500 Subject: [PATCH 078/182] style guide --- docs/dev/contributing.rst | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 094b7b19..585497bb 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -68,8 +68,8 @@ accepted. Code Contributions ------------------ -Steps -~~~~~ +Steps for Submitting Code +~~~~~~~~~~~~~~~~~~~~~~~~~ When contributing code, you'll want to follow this checklist: @@ -107,6 +107,30 @@ asking for help. Please also check the :ref:`early-feedback` section. +Code Style +~~~~~~~~~~ + +The Requests codebase uses the `PEP8`_ code style. + +In addition to the standards outlined in PEP8, we have a few guidelines: + +- Line-length can exceed 79 characters, to 100, when convenient. +- Line-length can exceed 100 characters, when doing otherwise would be *terribly* inconvenient. +- Always use single-quoted strings (e.g. ``'#flatearth'``), unless a single-quote occurs within the string. + +Additionally, one of the few style recommendations PEP8 makes for +`line continuations`_ lacks all sense of taste, and is not to be found within +the Requests codebase:: + + # Aligned with opening delimiter. + foo = long_function_name(var_one, var_two, + var_three, var_four) + +Just don't. Ever. + +.. _PEP8: https://www.python.org/dev/peps/pep-0008/ +.. _line continuations: https://www.python.org/dev/peps/pep-0008/#indentation + Documentation Contributions --------------------------- From f123f89d32d3258429cdecf33fbd20e68a746042 Mon Sep 17 00:00:00 2001 From: Dmitry Dygalo Date: Fri, 19 Feb 2016 10:32:23 +0100 Subject: [PATCH 079/182] Added unit tests for utils module --- requests/utils.py | 25 +++--- tests/test_utils.py | 204 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 205 insertions(+), 24 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index c5c3fd01..b4ceb1ef 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -14,9 +14,7 @@ import codecs import collections import io import os -import platform import re -import sys import socket import struct import warnings @@ -557,6 +555,7 @@ def should_bypass_proxies(url): return False + def get_environ_proxies(url): """Return a dict of environment proxies.""" if should_bypass_proxies(url): @@ -564,6 +563,7 @@ def get_environ_proxies(url): else: return getproxies() + def select_proxy(url, proxies): """Select a proxy for the url, if applicable. @@ -577,6 +577,7 @@ def select_proxy(url, proxies): proxy = proxies.get(urlparts.scheme) return proxy + def default_user_agent(name="python-requests"): """Return a string representing the default user agent.""" return '%s/%s' % (name, __version__) @@ -600,21 +601,19 @@ def parse_header_links(value): links = [] - replace_chars = " '\"" + replace_chars = ' \'"' - for val in re.split(", *<", value): + for val in re.split(', *<', value): try: - url, params = val.split(";", 1) + url, params = val.split(';', 1) except ValueError: url, params = val, '' - link = {} + link = {'url': url.strip('<> \'"')} - link["url"] = url.strip("<> '\"") - - for param in params.split(";"): + for param in params.split(';'): try: - key, value = param.split("=") + key, value = param.split('=') except ValueError: break @@ -661,8 +660,8 @@ def guess_json_utf(data): def prepend_scheme_if_needed(url, new_scheme): - '''Given a URL that may or may not have a scheme, prepend the given scheme. - Does not replace a present scheme with the one provided as an argument.''' + """Given a URL that may or may not have a scheme, prepend the given scheme. + Does not replace a present scheme with the one provided as an argument.""" scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme) # urlparse is a finicky beast, and sometimes decides that there isn't a @@ -693,8 +692,6 @@ def to_native_string(string, encoding='ascii'): string in the native string type, encoding and decoding where necessary. This assumes ASCII unless told otherwise. """ - out = None - if isinstance(string, builtin_str): out = string else: diff --git a/tests/test_utils.py b/tests/test_utils.py index 5a50e366..3d9d3936 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,15 +1,20 @@ # coding: utf-8 -import os from io import BytesIO import pytest from requests import compat +from requests.structures import CaseInsensitiveDict from requests.utils import ( address_in_network, dotted_netmask, - get_auth_from_url, get_encodings_from_content, - get_environ_proxies, guess_filename, - is_ipv4_address, is_valid_cidr, requote_uri, - select_proxy, super_len) + get_auth_from_url, get_encoding_from_headers, + get_encodings_from_content, get_environ_proxies, + guess_filename, guess_json_utf, is_ipv4_address, + is_valid_cidr, iter_slices, parse_dict_header, + parse_header_links, prepend_scheme_if_needed, + requote_uri, select_proxy, super_len, + to_key_val_list, to_native_string, + unquote_header_value, unquote_unreserved, + urldefragauth) from .compat import StringIO, cStringIO @@ -33,16 +38,63 @@ class TestSuperLen: s.write('foobarbogus') assert super_len(s) == 0 + def test_string(self): + assert super_len('Test') == 4 + + @pytest.mark.parametrize( + 'mode, warnings_num', ( + ('r', 1), + ('rb', 0), + )) + def test_file(self, tmpdir, mode, warnings_num, recwarn): + file_obj = tmpdir.join('test.txt') + file_obj.write('Test') + with file_obj.open(mode) as fd: + assert super_len(fd) == 4 + assert len(recwarn) == warnings_num + + +class TestToKeyValList: + + @pytest.mark.parametrize( + 'value, expected', ( + ([('key', 'val')], [('key', 'val')]), + ((('key', 'val'), ), [('key', 'val')]), + ({'key': 'val'}, [('key', 'val')]), + (None, None) + )) + def test_valid(self, value, expected): + assert to_key_val_list(value) == expected + + def test_invalid(self): + with pytest.raises(ValueError): + to_key_val_list('string') + + +class TestUnquoteHeaderValue: + + @pytest.mark.parametrize( + 'value, expected', ( + (None, None), + ('Test', 'Test'), + ('"Test"', 'Test'), + ('"Test\\\\"', 'Test\\'), + ('"\\\\Comp\\Res"', '\\Comp\\Res'), + )) + def test_valid(self, value, expected): + assert unquote_header_value(value) == expected + + def test_is_filename(self): + assert unquote_header_value('"\\\\Comp\\Res"', True) == '\\\\Comp\\Res' + class TestGetEnvironProxies: """Ensures that IP addresses are correctly matches with ranges in no_proxy variable.""" - @pytest.yield_fixture(scope='class', autouse=True, params=['no_proxy', 'NO_PROXY']) - def no_proxy(self, request): - os.environ[request.param] = '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1' - yield - del os.environ[request.param] + @pytest.fixture(autouse=True, params=['no_proxy', 'NO_PROXY']) + def no_proxy(self, request, monkeypatch): + monkeypatch.setenv(request.param, '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') @pytest.mark.parametrize( 'url', ( @@ -152,6 +204,21 @@ class TestContentEncodingDetection: assert get_encodings_from_content(content) == ['HTML5', 'HTML4', 'XML'] +class TestGuessJSONUTF: + + @pytest.mark.parametrize( + 'encoding', ( + 'utf-32', 'utf-8-sig', 'utf-16', 'utf-8', 'utf-16-be', 'utf-16-le', + 'utf-32-be', 'utf-32-le' + )) + def test_encoded(self, encoding): + data = '{}'.encode(encoding) + assert guess_json_utf(data) == encoding + + def test_bad_utf_like_encoding(self): + assert guess_json_utf(b'\x00\x00\x00\x00') is None + + USER = PASSWORD = "%!*'();:@&=+$,/?#[] " ENCODED_USER = compat.quote(USER, '') ENCODED_PASSWORD = compat.quote(PASSWORD, '') @@ -184,6 +251,10 @@ ENCODED_PASSWORD = compat.quote(PASSWORD, '') 'http://user:pass%23pass@complex.url.com/path?query=yes', ('user', 'pass#pass') ), + ( + 'http://complex.url.com/path?query=yes', + ('', '') + ), )) def test_get_auth_from_url(url, auth): assert get_auth_from_url(url) == auth @@ -208,6 +279,23 @@ def test_requote_uri_with_unquoted_percents(uri, expected): assert requote_uri(uri) == expected +@pytest.mark.parametrize( + 'uri, expected', ( + ( + # Illegal bytes + 'http://example.com/?a=%--', + 'http://example.com/?a=%--', + ), + ( + # Reserved characters + 'http://example.com/?a=%300', + 'http://example.com/?a=00', + ) + )) +def test_unquote_unreserved(uri, expected): + assert unquote_unreserved(uri) == expected + + @pytest.mark.parametrize( 'mask, expected', ( (8, '255.0.0.0'), @@ -229,3 +317,99 @@ def test_select_proxies(url, expected): proxies = {'http': 'http://http.proxy', 'http://some.host': 'http://some.host.proxy'} assert select_proxy(url, proxies) == expected + + +@pytest.mark.parametrize( + 'value, expected', ( + ('foo="is a fish", bar="as well"', {'foo': 'is a fish', 'bar': 'as well'}), + ('key_without_value', {'key_without_value': None}) + )) +def test_parse_dict_header(value, expected): + assert parse_dict_header(value) == expected + + +@pytest.mark.parametrize( + 'value, expected', ( + ( + CaseInsensitiveDict(), + None + ), + ( + CaseInsensitiveDict({'content-type': 'application/json; charset=utf-8'}), + 'utf-8' + ), + ( + CaseInsensitiveDict({'content-type': 'text/plain'}), + 'ISO-8859-1' + ), + )) +def test_get_encoding_from_headers(value, expected): + assert get_encoding_from_headers(value) == expected + + +@pytest.mark.parametrize( + 'value, length', ( + ('', 0), + ('T', 1), + ('Test', 4), + )) +def test_iter_slices(value, length): + assert len(list(iter_slices(value, 1))) == length + + +@pytest.mark.parametrize( + 'value, expected', ( + ( + '; rel=front; type="image/jpeg"', + [{'url': 'http:/.../front.jpeg', 'rel': 'front', 'type': 'image/jpeg'}] + ), + ( + '', + [{'url': 'http:/.../front.jpeg'}] + ), + ( + ';', + [{'url': 'http:/.../front.jpeg'}] + ), + ( + '; type="image/jpeg",;', + [ + {'url': 'http:/.../front.jpeg', 'type': 'image/jpeg'}, + {'url': 'http://.../back.jpeg'} + ] + ), + )) +def test_parse_header_links(value, expected): + assert parse_header_links(value) == expected + + +@pytest.mark.parametrize( + 'value, expected', ( + ('example.com/path', 'http://example.com/path'), + ('//example.com/path', 'http://example.com/path'), + )) +def test_prepend_scheme_if_needed(value, expected): + assert prepend_scheme_if_needed(value, 'http') == expected + + +@pytest.mark.parametrize( + 'value, expected', ( + ('T', 'T'), + (b'T', 'T'), + (u'T', 'T'), + )) +def test_to_native_string(value, expected): + assert to_native_string(value) == expected + + +@pytest.mark.parametrize( + 'url, expected', ( + ('http://u:p@example.com/path?a=1#test', 'http://example.com/path?a=1'), + ('http://example.com/path', 'http://example.com/path'), + ('//u:p@example.com/path', '//example.com/path'), + ('//example.com/path', '//example.com/path'), + ('example.com/path', '//example.com/path'), + ('scheme:u:p@example.com/path', 'scheme://example.com/path'), + )) +def test_urldefragauth(url, expected): + assert urldefragauth(url) == expected From 90ddeca70d7846e6a5b99a9c1fe1a2666b5ca47c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 04:42:54 -0500 Subject: [PATCH 080/182] updated style guide --- docs/dev/contributing.rst | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 585497bb..1c43cea7 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -126,7 +126,21 @@ the Requests codebase:: foo = long_function_name(var_one, var_two, var_three, var_four) -Just don't. Ever. +No. + +Docstrings are to follow the following syntaxes:: + + def the_earth_is_flat(): + """There is no curve! Also, human cloning centers.""" + +:: + + def well_documented_utility(): + """Lorem ipsum dolor sit amet, consectetur adipiscing elit, + sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. + Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris + nisi ut aliquip ex ea commodo consequat. + """ .. _PEP8: https://www.python.org/dev/peps/pep-0008/ .. _line continuations: https://www.python.org/dev/peps/pep-0008/#indentation From ee024423a5d586b2e2cfe773cc5c47a8dcf1db99 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 04:44:11 -0500 Subject: [PATCH 081/182] more pleasant wording about strong opinion --- docs/dev/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 1c43cea7..e4209366 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -126,7 +126,7 @@ the Requests codebase:: foo = long_function_name(var_one, var_two, var_three, var_four) -No. +No. Just don't. Please. Docstrings are to follow the following syntaxes:: From 29b191b5ea66f7d8e4f9a5db2cc6e0aec9c0d884 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 04:48:47 -0500 Subject: [PATCH 082/182] --- docs/dev/contributing.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index e4209366..85e443b2 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -142,6 +142,8 @@ Docstrings are to follow the following syntaxes:: nisi ut aliquip ex ea commodo consequat. """ +Thanks for helping to make the world a better place! + .. _PEP8: https://www.python.org/dev/peps/pep-0008/ .. _line continuations: https://www.python.org/dev/peps/pep-0008/#indentation From 8dc5f68bc1d9d28b79478f495be445ba048e8081 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 04:50:55 -0500 Subject: [PATCH 083/182] sp --- docs/dev/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 85e443b2..feb7e741 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -184,7 +184,7 @@ Feature Requests Requests is in a perpetual feature freeze, only the BDFL can add or approve of new features. The maintainers believe that Requests is a feature-complete -peice of software at this time. +piece of software at this time. One of the most important skills to have while maintaining a largely-used open source project is learning the ability to say "no" to suggested changes, From 1cf1699e37139daa37913e989c91fa130a2472fe Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 04:56:25 -0500 Subject: [PATCH 084/182] what's docstrung --- docs/dev/contributing.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index feb7e741..41159fb9 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -142,6 +142,9 @@ Docstrings are to follow the following syntaxes:: nisi ut aliquip ex ea commodo consequat. """ +All functions, methods, and classes are to contain docstrings. Object data +model methods (e.g. ``__repr__``) are usually an exception to this rule. + Thanks for helping to make the world a better place! .. _PEP8: https://www.python.org/dev/peps/pep-0008/ From f653b9ac58ad74441e118538619979edcf6749a5 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 05:03:04 -0500 Subject: [PATCH 085/182] sentences --- docs/dev/contributing.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 41159fb9..0c17c3b3 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -56,13 +56,14 @@ into a contribution that is not suitable for the project. Contribution Suitability ------------------------ -The project maintainer has the last word on whether or not a contribution is -suitable for Requests. All contributions will be considered, but from time -to time contributions will be rejected because they do not suit the project. +Our project maintainers have the last word on whether or not a contribution is +suitable for Requests. All contributions will be considered carefully, but from +time to time, contributions will be rejected because they do not suit the +current goals or needs of the project. -If your contribution is rejected, don't despair! So long as you followed these -guidelines, you'll have a much better chance of getting your next contribution -accepted. +If your contribution is rejected, don't despair! As long as you followed these +guidelines, you will have a much better chance of getting your next +contribution accepted. Code Contributions From b70136cf52e8dfd5a950fecfe2aee820dc7926ef Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 05:38:33 -0500 Subject: [PATCH 086/182] completely (docs) --- docs/dev/contributing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 0c17c3b3..82b4a45c 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -119,8 +119,8 @@ In addition to the standards outlined in PEP8, we have a few guidelines: - Line-length can exceed 100 characters, when doing otherwise would be *terribly* inconvenient. - Always use single-quoted strings (e.g. ``'#flatearth'``), unless a single-quote occurs within the string. -Additionally, one of the few style recommendations PEP8 makes for -`line continuations`_ lacks all sense of taste, and is not to be found within +Additionally, one of the styles that PEP8 recommends for `line continuations`_ +completely lacks all sense of taste, and is not to be permitted within the Requests codebase:: # Aligned with opening delimiter. From 14a83339a74929eae193fbbefd3668cfa8943780 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 05:41:17 -0500 Subject: [PATCH 087/182] words (docs) --- docs/dev/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 82b4a45c..f17af82c 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -144,7 +144,7 @@ Docstrings are to follow the following syntaxes:: """ All functions, methods, and classes are to contain docstrings. Object data -model methods (e.g. ``__repr__``) are usually an exception to this rule. +model methods (e.g. ``__repr__``) are typically the exception to this rule. Thanks for helping to make the world a better place! From d8bf59dde4bd47f07ef88a4161b4a652b06c7599 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 05:45:55 -0500 Subject: [PATCH 088/182] namedrop --- docs/dev/contributing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index f17af82c..1f692621 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -108,8 +108,8 @@ asking for help. Please also check the :ref:`early-feedback` section. -Code Style -~~~~~~~~~~ +Kenneth Reitz's Code Style +~~~~~~~~~~~~~~~~~~~~~~~~~~ The Requests codebase uses the `PEP8`_ code style. From 1b1f4ac77d778eb730790890ce764e5127758e02 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 06:08:38 -0500 Subject: [PATCH 089/182] cheat code --- docs/_static/konami.js | 116 +++++++++++++++++++++++++++++++++++++ docs/_templates/hacks.html | 9 ++- docs/index.rst | 9 ++- 3 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 docs/_static/konami.js diff --git a/docs/_static/konami.js b/docs/_static/konami.js new file mode 100644 index 00000000..d72cf9df --- /dev/null +++ b/docs/_static/konami.js @@ -0,0 +1,116 @@ +/* + * Konami-JS ~ + * :: Now with support for touch events and multiple instances for + * :: those situations that call for multiple easter eggs! + * Code: http://konami-js.googlecode.com/ + * Examples: http://www.snaptortoise.com/konami-js + * Copyright (c) 2009 George Mandis (georgemandis.com, snaptortoise.com) + * Version: 1.4.2 (9/2/2013) + * Licensed under the MIT License (http://opensource.org/licenses/MIT) + * Tested in: Safari 4+, Google Chrome 4+, Firefox 3+, IE7+, Mobile Safari 2.2.1 and Dolphin Browser + */ + +var Konami = function (callback) { + var konami = { + addEvent: function (obj, type, fn, ref_obj) { + if (obj.addEventListener) + obj.addEventListener(type, fn, false); + else if (obj.attachEvent) { + // IE + obj["e" + type + fn] = fn; + obj[type + fn] = function () { + obj["e" + type + fn](window.event, ref_obj); + }; + obj.attachEvent("on" + type, obj[type + fn]); + } + }, + input: "", + pattern: "38384040373937396665", + load: function (link) { + this.addEvent(document, "keydown", function (e, ref_obj) { + if (ref_obj) konami = ref_obj; // IE + konami.input += e ? e.keyCode : event.keyCode; + if (konami.input.length > konami.pattern.length) + konami.input = konami.input.substr((konami.input.length - konami.pattern.length)); + if (konami.input == konami.pattern) { + konami.code(link); + konami.input = ""; + e.preventDefault(); + return false; + } + }, this); + this.iphone.load(link); + }, + code: function (link) { + window.location = link + }, + iphone: { + start_x: 0, + start_y: 0, + stop_x: 0, + stop_y: 0, + tapTolerance: 8, + capture: false, + orig_keys: "", + keys: ["UP", "UP", "DOWN", "DOWN", "LEFT", "RIGHT", "LEFT", "RIGHT", "TAP", "TAP"], + code: function (link) { + konami.code(link); + }, + touchCapture: function(evt) { + konami.iphone.start_x = evt.changedTouches[0].pageX; + konami.iphone.start_y = evt.changedTouches[0].pageY; + konami.iphone.capture = true; + }, + load: function (link) { + this.orig_keys = this.keys; + konami.addEvent(document, "touchmove", function (e) { + if (e.touches.length == 1 && konami.iphone.capture == true) { + var touch = e.touches[0]; + konami.iphone.stop_x = touch.pageX; + konami.iphone.stop_y = touch.pageY; + konami.iphone.check_direction(); + } + }); + konami.addEvent(document, "touchend", function (evt) { + konami.touchCapture(evt); + konami.iphone.check_direction(link); + }, false); + konami.addEvent(document, "touchstart", function (evt) { + konami.touchCapture(evt); + }); + }, + check_direction: function (link) { + var x_magnitude = Math.abs(this.start_x - this.stop_x); + var y_magnitude = Math.abs(this.start_y - this.stop_y); + var hasMoved = (x_magnitude > this.tapTolerance || y_magnitude > this.tapTolerance); + var result; + if (this.capture === true && hasMoved) { + this.capture = false; + var x = ((this.start_x - this.stop_x) < 0) ? "RIGHT" : "LEFT"; + var y = ((this.start_y - this.stop_y) < 0) ? "DOWN" : "UP"; + var result = (x_magnitude > y_magnitude) ? x : y; + } + else if (this.capture === false && !hasMoved) { + result = (this.tap == true) ? "TAP" : result; + result = "TAP"; + } + if (result) { + if (result == this.keys[0]) this.keys = this.keys.slice(1, this.keys.length); + else this.keys = this.orig_keys; + } + if (this.keys.length == 0) { + this.keys = this.orig_keys; + this.code(link); + } + } + } + } + + typeof callback === "string" && konami.load(callback); + if (typeof callback === "function") { + konami.code = callback; + konami.load(); + } + + return konami; +}; diff --git a/docs/_templates/hacks.html b/docs/_templates/hacks.html index ba032452..ca9865d6 100644 --- a/docs/_templates/hacks.html +++ b/docs/_templates/hacks.html @@ -44,4 +44,11 @@ - \ No newline at end of file + + + + + + \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 0df56107..75983c37 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -124,8 +124,8 @@ Requests ecosystem and community. community/updates community/release-process -API Documentation ------------------ +API Documentation / Guide +------------------------- If you are looking for information on a specific function, class or method, this part of the documentation is for you. @@ -143,9 +143,12 @@ If you want to contribute to the project, this part of the documentation is for you. .. toctree:: - :maxdepth: 1 + :maxdepth: 2 dev/contributing dev/philosophy dev/todo dev/authors + +There are no more guides. You are now guideless. +Good luck. From ca303cda0a4664bf925ba2eecdd4c5abf1987443 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 06:14:51 -0500 Subject: [PATCH 090/182] letters --- docs/index.rst | 2 +- docs/user/install.rst | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 75983c37..cc4d4510 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -143,7 +143,7 @@ If you want to contribute to the project, this part of the documentation is for you. .. toctree:: - :maxdepth: 2 + :maxdepth: 3 dev/contributing dev/philosophy diff --git a/docs/user/install.rst b/docs/user/install.rst index b14ee310..c3f0084e 100644 --- a/docs/user/install.rst +++ b/docs/user/install.rst @@ -7,8 +7,8 @@ This part of the documentation covers the installation of Requests. The first step to using any software package is getting it properly installed. -Pip Install ------------ +Pip Install Requests +-------------------- To install Requests, simply run this simple command in your terminal of choice:: @@ -18,8 +18,8 @@ If you don't have `pip `_ installed (tisk tisk!), `this Python installation guide `_ can guide you through the process. -Get the Code ------------- +Get the Source Code +------------------- Requests is actively developed on GitHub, where the code is `always available `_. From a7b5e3191d1b121e452e83798287209d12ade7ee Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 08:51:36 -0500 Subject: [PATCH 091/182] pass go, yay $200 --- docs/dev/contributing.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 1f692621..24f4b9f7 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -133,6 +133,7 @@ Docstrings are to follow the following syntaxes:: def the_earth_is_flat(): """There is no curve! Also, human cloning centers.""" + pass :: @@ -142,6 +143,7 @@ Docstrings are to follow the following syntaxes:: Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. """ + pass All functions, methods, and classes are to contain docstrings. Object data model methods (e.g. ``__repr__``) are typically the exception to this rule. From fc1e9b46a197b8412fa3fdb7d2bcb5e0f62f68a5 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 09:08:29 -0500 Subject: [PATCH 092/182] lyrics, er, docstrings --- docs/dev/contributing.rst | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 24f4b9f7..302d1bd6 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -132,16 +132,20 @@ No. Just don't. Please. Docstrings are to follow the following syntaxes:: def the_earth_is_flat(): - """There is no curve! Also, human cloning centers.""" + """NASA divided up the seas into thirty-three degrees.""" pass :: - def well_documented_utility(): - """Lorem ipsum dolor sit amet, consectetur adipiscing elit, - sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. - Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris - nisi ut aliquip ex ea commodo consequat. + def fibonacci_spiral_tool(): + """With my feet upon the ground I lose myself / between the sounds and open + wide to suck it in. / I feel it move across my skin. / I'm reaching up and + reaching out. / I'm reaching for the random or whatever will bewilder me. / + Whatever will bewilder me. /And following our will and wind we may just go + where no one's been. / We'll ride the spiral to the end and may just go + where no one's been. + + Spiral out. Keep going... """ pass From b366ef039cf2d9cafaaeed063d44ebef2d7f801c Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 09:11:09 -0500 Subject: [PATCH 093/182] improved presentation of style guide --- docs/dev/contributing.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 302d1bd6..fb6eb874 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -138,12 +138,13 @@ Docstrings are to follow the following syntaxes:: :: def fibonacci_spiral_tool(): - """With my feet upon the ground I lose myself / between the sounds and open - wide to suck it in. / I feel it move across my skin. / I'm reaching up and - reaching out. / I'm reaching for the random or whatever will bewilder me. / - Whatever will bewilder me. /And following our will and wind we may just go - where no one's been. / We'll ride the spiral to the end and may just go - where no one's been. + """With my feet upon the ground I lose myself / between the sounds + and open wide to suck it in. / I feel it move across my skin. / I'm + reaching up and reaching out. / I'm reaching for the random or + whatever will bewilder me. / Whatever will bewilder me. / And + following our will and wind we may just go where no one's been. / + We'll ride the spiral to the end and may just go where no one's + been. Spiral out. Keep going... """ From 75096a167cd6c3293223995d9207ceb96f254fd6 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 19 Feb 2016 09:26:21 -0500 Subject: [PATCH 094/182] transcendental meditation --- docs/dev/contributing.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index fb6eb874..4b98a945 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -108,8 +108,8 @@ asking for help. Please also check the :ref:`early-feedback` section. -Kenneth Reitz's Code Style -~~~~~~~~~~~~~~~~~~~~~~~~~~ +Kenneth Reitz's Code Style™ +~~~~~~~~~~~~~~~~~~~~~~~~~~~ The Requests codebase uses the `PEP8`_ code style. From 41c26ce1f54278954575df015d861b4dd5cb8894 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 21 Feb 2016 20:32:36 -0500 Subject: [PATCH 095/182] improvements to index --- docs/index.rst | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index cc4d4510..5eb643e1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -38,8 +38,8 @@ URLs, or to form-encode your POST data. Keep-alive and HTTP connection pooling are 100% automatic, powered by `urllib3 `_, which is embedded within Requests. -Testimonials ------------- +User Testimonials +----------------- Her Majesty's Government, Amazon, Google, Twilio, Runscope, Mozilla, Heroku, PayPal, NPR, Obama for America, Transifex, Native Instruments, The Washington @@ -65,8 +65,8 @@ Institutions that prefer to be unnamed claim to use Requests internally. Requests is one of the most downloaded Python packages of all time, pulling in over 7,000,000 downloads every month. All the cool kids are doing it! -Feature Support ---------------- +Supported Features +------------------ Requests is ready for today's web. @@ -90,8 +90,8 @@ Requests is ready for today's web. Requests supports Python 2.6 — 3.5, and runs great on PyPy. -User Guide ----------- +The User Guide +-------------- This part of the documentation, which is mostly prose, begins with some background information about Requests, then focuses on step-by-step @@ -107,8 +107,8 @@ instructions for getting the most out of Requests. user/authentication -Community Guide ------------------ +The Community Guide +------------------- This part of the documentation, which is mostly prose, details the Requests ecosystem and community. @@ -124,10 +124,10 @@ Requests ecosystem and community. community/updates community/release-process -API Documentation / Guide -------------------------- +The API Documentation / Guide +----------------------------- -If you are looking for information on a specific function, class or method, +If you are looking for information on a specific function, class, or method, this part of the documentation is for you. .. toctree:: @@ -136,8 +136,8 @@ this part of the documentation is for you. api -Contributor Guide ------------------ +The Contributor Guide +--------------------- If you want to contribute to the project, this part of the documentation is for you. From 5f06b8196e8a652101b32fdffcd0d10e8b961fd2 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 21 Feb 2016 20:32:42 -0500 Subject: [PATCH 096/182] updated sidebars --- docs/_templates/sidebarintro.html | 15 ++++++++++++++- docs/_templates/sidebarlogo.html | 16 ++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html index 38661e00..69ef6685 100644 --- a/docs/_templates/sidebarintro.html +++ b/docs/_templates/sidebarintro.html @@ -14,7 +14,7 @@ human beings.

-

Stay Informed

+

Get Updates

Receive updates on new releases and upcoming projects.

-

\ No newline at end of file +

+ +

Other Projects

+ +

MoreKenneth Reitz projects:

+ \ No newline at end of file From e47e80b92f354efe5db7049b40f9cadc4ea7566e Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 21 Feb 2016 20:34:15 -0500 Subject: [PATCH 097/182] &bsp; --- docs/_templates/sidebarlogo.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index c05ab3e7..7bcd4735 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -26,7 +26,7 @@

Other Projects

-

MoreKenneth Reitz projects:

+

More Kenneth Reitz projects:

  • pep8.org
  • The Python Guide
    • From f2ec732ce8bd339187cc01f3c5290d4fc19ed0d1 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sun, 21 Feb 2016 20:45:45 -0500 Subject: [PATCH 098/182] stay informed --- docs/_templates/sidebarintro.html | 2 +- docs/_templates/sidebarlogo.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html index 69ef6685..3fa77a1d 100644 --- a/docs/_templates/sidebarintro.html +++ b/docs/_templates/sidebarintro.html @@ -14,7 +14,7 @@ human beings.

    -

    Get Updates

    +

    Stay Informed

    Receive updates on new releases and upcoming projects.