diff --git a/AUTHORS.rst b/AUTHORS.rst index 3eeb1cff..eeccd74d 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -166,3 +166,5 @@ Patches and Suggestions - Brian Samek (`@bsamek `_) - Dmitry Dygalo (`@Stranger6667 `_) - Tomáš Heger (`@geckon `_) +- piotrjurkiewicz +- Jesse Shapiro (`@haikuginger `_) diff --git a/HISTORY.rst b/HISTORY.rst index f8c1a545..8913c8c0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,32 @@ Release History --------------- +2.10.0 (2016-04-29) ++++++++++++++++++++ + +**New Features** + +- SOCKS Proxy Support! (requires PySocks; $ pip install requests[socks]) + +**Miscellaneous** + +- Updated bundled urllib3 to 1.15.1. + +2.9.2 (2016-04-29) +++++++++++++++++++ + +**Improvements** + +- Change built-in CaseInsensitiveDict (used for headers) to use OrderedDict + as its underlying datastore. + +**Bugfixes** + +- Don't use redirect_cache if allow_redirects=False +- When passed objects that throw exceptions from ``tell()``, send them via + chunked transfer encoding instead of failing. +- Raise a ProxyError for proxy related connection issues. + 2.9.1 (2015-12-21) ++++++++++++++++++ diff --git a/Makefile b/Makefile index dea33a3f..a8289712 100644 --- a/Makefile +++ b/Makefile @@ -6,10 +6,10 @@ init: test: # This runs all of the tests. To run an individual test, run py.test with # the -k flag, like "py.test -k test_path_is_not_double_encoded" - py.test test_requests.py + py.test tests coverage: - py.test --verbose --cov-report term --cov=requests test_requests.py + py.test --verbose --cov-report term --cov=requests tests ci: init py.test --junitxml=junit.xml @@ -28,7 +28,8 @@ chardet: publish: python setup.py register python setup.py sdist upload - python setup.py bdist_wheel upload + python setup.py bdist_wheel --universal upload + rm -fr build dist .egg requests.egg-info docs-init: diff --git a/README.rst b/README.rst index 99d30e72..9fe548d2 100644 --- a/README.rst +++ b/README.rst @@ -7,41 +7,45 @@ Requests: HTTP for Humans .. image:: https://img.shields.io/pypi/dm/requests.svg :target: https://pypi.python.org/pypi/requests +Requests is the only *Non-GMO* HTTP library for Python, safe for human +consumption. +**Warning:** Recreational use of other HTTP libraries may result in dangerous side-effects, +including: security vulnerabilities, verbose code, reinventing the wheel, +constantly reading documentation, depression, headaches, or even death. - -Requests is an Apache2 Licensed HTTP library, written in Python, for human -beings. - -Most existing Python modules for sending HTTP requests are extremely -verbose and cumbersome. Python's builtin urllib2 module provides most of -the HTTP capabilities you should need, but the api is thoroughly broken. -It requires an enormous amount of work (even method overrides) to -perform the simplest of tasks. - -Things shouldn't be this way. Not in Python. +Behold, the power of Requests: .. code-block:: python - >>> r = requests.get('https://api.github.com', auth=('user', 'pass')) + >>> r = requests.get('https://api.github.com/user', auth=('user', 'pass')) >>> r.status_code - 204 + 200 >>> r.headers['content-type'] - 'application/json' + 'application/json; charset=utf8' + >>> r.encoding + 'utf-8' >>> r.text - ... + u'{"type":"User"...' + >>> r.json() + {u'disk_usage': 368627, u'private_gists': 484, ...} -See `the same code, without Requests `_. +See `the similar code, sans Requests `_. -Requests allow you to send HTTP/1.1 requests. You can add headers, form data, -multipart files, and parameters with simple Python dictionaries, and access the -response data in the same way. It's powered by httplib and `urllib3 -`_, but it does all the hard work and crazy -hacks for you. +Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, without the +need for manual labor. There's no need to manually add query strings to your +URLs, or to form-encode your POST data. Keep-alive and HTTP connection pooling +are 100% automatic, powered by `urllib3 `_, +which is embedded within Requests. +Besides, all the cool kids are doing it. Requests is one of the most +downloaded Python packages of all time, pulling in over 7,000,000 downloads +every month. You don't want to be left out! -Features --------- +Feature Support +--------------- + +Requests is ready for today's web. - International Domains and URLs - Keep-Alive & Connection Pooling @@ -50,12 +54,17 @@ Features - Basic/Digest Authentication - Elegant Key/Value Cookies - Automatic Decompression +- Automatic Content Decoding - Unicode Response Bodies - Multipart File Uploads +- HTTP(S) Proxy Support - Connection Timeouts +- Streaming Downloads +- ``.netrc`` Support +- Chunked Requests - Thread-safety -- HTTP(S) proxy support +Requests supports Python 2.6 — 3.5, and runs great on PyPy. Installation ------------ @@ -65,16 +74,18 @@ To install Requests, simply: .. code-block:: bash $ pip install requests + ✨🍰✨ +Satisfaction, guaranteed. Documentation ------------- -Documentation is available at http://docs.python-requests.org/. +Fantastic documentation is available at http://docs.python-requests.org/, for a limited time only. -Contribute ----------- +How to Contribute +----------------- #. Check for open issues or open a fresh issue to start a discussion around a feature idea or a bug. There is a `Contributor Friendly`_ tag for issues that should be ideal for people who are not very familiar with the codebase yet. #. Fork `the repository`_ on GitHub to start making your changes to the **master** branch (or branch off of it). diff --git a/docs/MANIFEST.in b/docs/MANIFEST.in deleted file mode 100644 index fb1021bf..00000000 --- a/docs/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include HISTORY.rst README.rst LICENSE \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile index 946ba445..08a2acf6 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -7,6 +7,11 @@ SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter @@ -14,8 +19,7 @@ ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - +.PHONY: help help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @@ -25,53 +29,66 @@ help: @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" +.PHONY: clean clean: - -rm -rf $(BUILDDIR)/* + rm -rf $(BUILDDIR)/* +.PHONY: html html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." +.PHONY: dirhtml dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." +.PHONY: singlehtml singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." +.PHONY: pickle pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." +.PHONY: json json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." +.PHONY: htmlhelp htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." +.PHONY: qthelp qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @@ -81,6 +98,16 @@ qthelp: @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Requests.qhc" +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @@ -90,11 +117,13 @@ devhelp: @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Requests" @echo "# devhelp" +.PHONY: epub epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." +.PHONY: latex latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @@ -102,22 +131,33 @@ latex: @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." +.PHONY: latexpdf latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." - make -C $(BUILDDIR)/latex all-pdf + $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." +.PHONY: man man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." +.PHONY: texinfo texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @@ -125,29 +165,52 @@ texinfo: @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." +.PHONY: info info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." +.PHONY: gettext gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." +.PHONY: changes changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." +.PHONY: linkcheck linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." +.PHONY: doctest doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/_static/konami.js b/docs/_static/konami.js new file mode 100644 index 00000000..d72cf9df --- /dev/null +++ b/docs/_static/konami.js @@ -0,0 +1,116 @@ +/* + * Konami-JS ~ + * :: Now with support for touch events and multiple instances for + * :: those situations that call for multiple easter eggs! + * Code: http://konami-js.googlecode.com/ + * Examples: http://www.snaptortoise.com/konami-js + * Copyright (c) 2009 George Mandis (georgemandis.com, snaptortoise.com) + * Version: 1.4.2 (9/2/2013) + * Licensed under the MIT License (http://opensource.org/licenses/MIT) + * Tested in: Safari 4+, Google Chrome 4+, Firefox 3+, IE7+, Mobile Safari 2.2.1 and Dolphin Browser + */ + +var Konami = function (callback) { + var konami = { + addEvent: function (obj, type, fn, ref_obj) { + if (obj.addEventListener) + obj.addEventListener(type, fn, false); + else if (obj.attachEvent) { + // IE + obj["e" + type + fn] = fn; + obj[type + fn] = function () { + obj["e" + type + fn](window.event, ref_obj); + }; + obj.attachEvent("on" + type, obj[type + fn]); + } + }, + input: "", + pattern: "38384040373937396665", + load: function (link) { + this.addEvent(document, "keydown", function (e, ref_obj) { + if (ref_obj) konami = ref_obj; // IE + konami.input += e ? e.keyCode : event.keyCode; + if (konami.input.length > konami.pattern.length) + konami.input = konami.input.substr((konami.input.length - konami.pattern.length)); + if (konami.input == konami.pattern) { + konami.code(link); + konami.input = ""; + e.preventDefault(); + return false; + } + }, this); + this.iphone.load(link); + }, + code: function (link) { + window.location = link + }, + iphone: { + start_x: 0, + start_y: 0, + stop_x: 0, + stop_y: 0, + tapTolerance: 8, + capture: false, + orig_keys: "", + keys: ["UP", "UP", "DOWN", "DOWN", "LEFT", "RIGHT", "LEFT", "RIGHT", "TAP", "TAP"], + code: function (link) { + konami.code(link); + }, + touchCapture: function(evt) { + konami.iphone.start_x = evt.changedTouches[0].pageX; + konami.iphone.start_y = evt.changedTouches[0].pageY; + konami.iphone.capture = true; + }, + load: function (link) { + this.orig_keys = this.keys; + konami.addEvent(document, "touchmove", function (e) { + if (e.touches.length == 1 && konami.iphone.capture == true) { + var touch = e.touches[0]; + konami.iphone.stop_x = touch.pageX; + konami.iphone.stop_y = touch.pageY; + konami.iphone.check_direction(); + } + }); + konami.addEvent(document, "touchend", function (evt) { + konami.touchCapture(evt); + konami.iphone.check_direction(link); + }, false); + konami.addEvent(document, "touchstart", function (evt) { + konami.touchCapture(evt); + }); + }, + check_direction: function (link) { + var x_magnitude = Math.abs(this.start_x - this.stop_x); + var y_magnitude = Math.abs(this.start_y - this.stop_y); + var hasMoved = (x_magnitude > this.tapTolerance || y_magnitude > this.tapTolerance); + var result; + if (this.capture === true && hasMoved) { + this.capture = false; + var x = ((this.start_x - this.stop_x) < 0) ? "RIGHT" : "LEFT"; + var y = ((this.start_y - this.stop_y) < 0) ? "DOWN" : "UP"; + var result = (x_magnitude > y_magnitude) ? x : y; + } + else if (this.capture === false && !hasMoved) { + result = (this.tap == true) ? "TAP" : result; + result = "TAP"; + } + if (result) { + if (result == this.keys[0]) this.keys = this.keys.slice(1, this.keys.length); + else this.keys = this.orig_keys; + } + if (this.keys.length == 0) { + this.keys = this.orig_keys; + this.code(link); + } + } + } + } + + typeof callback === "string" && konami.load(callback); + if (typeof callback === "function") { + konami.code = callback; + konami.load(); + } + + return konami; +}; diff --git a/docs/_templates/hacks.html b/docs/_templates/hacks.html new file mode 100644 index 00000000..f9fc96cb --- /dev/null +++ b/docs/_templates/hacks.html @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html deleted file mode 100644 index 3f435497..00000000 --- a/docs/_templates/layout.html +++ /dev/null @@ -1,86 +0,0 @@ -{%- extends "basic/layout.html" %} -{%- block extrahead %} - - {{ super() }} - - {% if theme_touch_icon %} - - {% endif %} - - - - - -{% endblock %} -{%- block relbar2 %}{% endblock %} -{%- block footer %} -
- © Copyright {{ copyright }}. -
- - Fork me on GitHub - - - - - - - - - - - - - -{%- endblock %} diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html index d2cbfe9d..7dbd74c8 100644 --- a/docs/_templates/sidebarintro.html +++ b/docs/_templates/sidebarintro.html @@ -14,30 +14,30 @@ human beings.

- -

- Buy Requests Pro -

- - -

Get Updates

+

Stay Informed

Receive updates on new releases and upcoming projects.

-

Subscribe to Newsletter

+

+

-

Translations

+

Join Mailing List.

+ +

Other Projects

+ +

More Kenneth Reitz projects:

+

Useful Links

+ + +

Translations

+ + + diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index 928cd2fd..1e295f81 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -14,13 +14,40 @@ development release.

-

- Buy Requests Pro -

- -

Get Updates

+

Stay Informed

Receive updates on new releases and upcoming projects.

-

Subscribe to Newsletter

+

Join Mailing List.

+ +

+ +

+ +

Other Projects

+ +

More Kenneth Reitz projects:

+ +

Translations

+ + + diff --git a/docs/api.rst b/docs/api.rst index 380ec6fc..6ba37784 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -25,9 +25,30 @@ They all return an instance of the :class:`Response ` object. .. autofunction:: patch .. autofunction:: delete +Exceptions +---------- + +.. autoexception:: requests.RequestException +.. autoexception:: requests.ConnectionError +.. autoexception:: requests.HTTPError +.. autoexception:: requests.URLRequired +.. autoexception:: requests.TooManyRedirects +.. autoexception:: requests.ConnectTimeout +.. autoexception:: requests.ReadTimeout +.. autoexception:: requests.Timeout + + +Request Sessions +---------------- + +.. _sessionapi: + +.. autoclass:: Session + :inherited-members: + Lower-Level Classes -~~~~~~~~~~~~~~~~~~~ +------------------- .. autoclass:: requests.Request :inherited-members: @@ -35,10 +56,11 @@ Lower-Level Classes .. autoclass:: Response :inherited-members: -Request Sessions ----------------- -.. autoclass:: Session +Lower-Lower-Level Classes +------------------------- + +.. autoclass:: requests.PreparedRequest :inherited-members: .. autoclass:: requests.adapters.HTTPAdapter @@ -52,39 +74,20 @@ Authentication .. autoclass:: requests.auth.HTTPProxyAuth .. autoclass:: requests.auth.HTTPDigestAuth -Exceptions -~~~~~~~~~~ - -.. autoexception:: requests.exceptions.RequestException -.. autoexception:: requests.exceptions.ConnectionError -.. autoexception:: requests.exceptions.HTTPError -.. autoexception:: requests.exceptions.URLRequired -.. autoexception:: requests.exceptions.TooManyRedirects -.. autoexception:: requests.exceptions.ConnectTimeout -.. autoexception:: requests.exceptions.ReadTimeout -.. autoexception:: requests.exceptions.Timeout -Status Code Lookup -~~~~~~~~~~~~~~~~~~ +Encodings +--------- -.. autofunction:: requests.codes +.. autofunction:: requests.utils.get_encodings_from_content +.. autofunction:: requests.utils.get_encoding_from_headers +.. autofunction:: requests.utils.get_unicode_from_response -:: - - >>> requests.codes['temporary_redirect'] - 307 - - >>> requests.codes.teapot - 418 - - >>> requests.codes['\o/'] - 200 .. _api-cookies: Cookies -~~~~~~~ +------- .. autofunction:: requests.utils.dict_from_cookiejar .. autofunction:: requests.utils.cookiejar_from_dict @@ -97,33 +100,23 @@ Cookies :inherited-members: -Encodings -~~~~~~~~~ -.. autofunction:: requests.utils.get_encodings_from_content -.. autofunction:: requests.utils.get_encoding_from_headers -.. autofunction:: requests.utils.get_unicode_from_response +Status Code Lookup +------------------ +.. autoclass:: requests.codes -Classes -~~~~~~~ +:: -.. autoclass:: requests.Response - :inherited-members: + >>> requests.codes['temporary_redirect'] + 307 -.. autoclass:: requests.Request - :inherited-members: + >>> requests.codes.teapot + 418 -.. autoclass:: requests.PreparedRequest - :inherited-members: + >>> requests.codes['\o/'] + 200 -.. _sessionapi: - -.. autoclass:: requests.Session - :inherited-members: - -.. autoclass:: requests.adapters.HTTPAdapter - :inherited-members: Migrating to 1.x diff --git a/docs/community/recommended.rst b/docs/community/recommended.rst index 99a16b9e..ae2ae5eb 100644 --- a/docs/community/recommended.rst +++ b/docs/community/recommended.rst @@ -22,7 +22,7 @@ CacheControl makes your web requests substantially more efficient, and should be used whenever you're making a lot of web requests. -.. _CacheControl: https://cachecontrol.readthedocs.org/en/latest/ +.. _CacheControl: https://cachecontrol.readthedocs.io/en/latest/ Requests-Toolbelt ----------------- @@ -32,7 +32,7 @@ but do not belong in Requests proper. This library is actively maintained by members of the Requests core team, and reflects the functionality most requested by users within the community. -.. _Requests-Toolbelt: http://toolbelt.readthedocs.org/en/latest/index.html +.. _Requests-Toolbelt: http://toolbelt.readthedocs.io/en/latest/index.html Requests-OAuthlib ----------------- @@ -42,7 +42,7 @@ automatically. This is useful for the large number of websites that use OAuth to provide authentication. It also provides a lot of tweaks that handle ways that specific OAuth providers differ from the standard specifications. -.. _requests-oauthlib: https://requests-oauthlib.readthedocs.org/en/latest/ +.. _requests-oauthlib: https://requests-oauthlib.readthedocs.io/en/latest/ Betamax diff --git a/docs/conf.py b/docs/conf.py index a84469db..fb8a01e0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- # # Requests documentation build configuration file, created by -# sphinx-quickstart on Sun Feb 13 23:54:25 2011. +# sphinx-quickstart on Fri Feb 19 00:05:47 2016. # -# This file is execfile()d with the current directory set to its containing dir. +# This file is execfile()d with the current directory set to its +# containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. @@ -11,34 +12,43 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import sys +import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# Insert Requests' path into the system. sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.abspath('_themes')) + import requests from requests import __version__ -import alabaster -# -- General configuration ----------------------------------------------------- +# -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx', - 'alabaster' + 'sphinx.ext.todo', + 'sphinx.ext.viewcode', ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] -# The suffix of source filenames. +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The encoding of source files. @@ -50,6 +60,7 @@ master_doc = 'index' # General information about the project. project = u'Requests' copyright = u'2016. A Kenneth Reitz Project' +author = u'Kenneth Reitz' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -58,11 +69,14 @@ copyright = u'2016. A tag referring to it. The value of this option must be the @@ -181,23 +208,45 @@ html_show_sphinx = False # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + # Output file base name for HTML help builder. htmlhelp_basename = 'Requestsdoc' +# -- Options for LaTeX output --------------------------------------------- -# -- Options for LaTeX output -------------------------------------------------- - -# The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} # Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'Requests.tex', u'Requests Documentation', - u'Kenneth Reitz', 'manual'), + (master_doc, 'Requests.tex', u'Requests Documentation', + u'Kenneth Reitz', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -214,9 +263,6 @@ latex_documents = [ # If true, show URL addresses after external links. #latex_show_urls = False -# Additional stuff for the LaTeX preamble. -#latex_preamble = '' - # Documents to append as an appendix to all manuals. #latex_appendices = [] @@ -224,33 +270,110 @@ latex_documents = [ #latex_domain_indices = True -# -- Options for manual page output -------------------------------------------- +# -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'requests', u'Requests Documentation', - [u'Kenneth Reitz'], 1) + (master_doc, 'requests', u'Requests Documentation', + [author], 1) ] # If true, show URL addresses after external links. #man_show_urls = False -# -- Options for Texinfo output ------------------------------------------------ + +# -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'Requests', u'Requests Documentation', u'Kenneth Reitz', - 'Requests', 'One line description of project.', 'Miscellaneous'), + (master_doc, 'Requests', u'Requests Documentation', + author, 'Requests', 'One line description of project.', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. -texinfo_appendices = [] +#texinfo_appendices = [] -sys.path.append(os.path.abspath('_themes')) -html_theme_path =[alabaster.get_path()] -html_theme = 'alabaster' +# If false, no module index is generated. +#texinfo_domain_indices = True -intersphinx_mapping = {'urllib3': ('http://urllib3.readthedocs.org/en/latest', None)} +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False + + +# -- Options for Epub output ---------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project +epub_author = author +epub_publisher = author +epub_copyright = copyright + +# The basename for the epub file. It defaults to the project name. +#epub_basename = project + +# The HTML theme for the epub output. Since the default themes are not +# optimized for small screen space, using the same theme for HTML and epub +# output is usually not wise. This defaults to 'epub', a theme designed to save +# visual space. +#epub_theme = 'epub' + +# The language of the text. It defaults to the language option +# or 'en' if the language is not set. +#epub_language = '' + +# The scheme of the identifier. Typical schemes are ISBN or URL. +#epub_scheme = '' + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +#epub_identifier = '' + +# A unique identification for the text. +#epub_uid = '' + +# A tuple containing the cover image and cover page html template filenames. +#epub_cover = () + +# A sequence of (type, uri, title) tuples for the guide element of content.opf. +#epub_guide = () + +# HTML files that should be inserted before the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_pre_files = [] + +# HTML files that should be inserted after the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_post_files = [] + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + +# The depth of the table of contents in toc.ncx. +#epub_tocdepth = 3 + +# Allow duplicate toc entries. +#epub_tocdup = True + +# Choose between 'default' and 'includehidden'. +#epub_tocscope = 'default' + +# Fix unsupported image types using the Pillow. +#epub_fix_images = False + +# Scale large images. +#epub_max_image_width = 0 + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#epub_show_urls = 'inline' + +# If false, no index is generated. +#epub_use_index = True + +intersphinx_mapping = {'urllib3': ('http://urllib3.readthedocs.io/en/latest', None)} diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index 075d042a..93181dad 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -3,37 +3,40 @@ Contributor's Guide =================== -If you're reading this you're probably interested in contributing to -Requests. First, We'd like to say: thank you! Open source projects -live-and-die based on the support they receive from others, and the fact that -you're even considering supporting Requests is very generous of -you. +If you're reading this, you're probably interested in contributing to Requests. +Thank you very much! Open source projects live-and-die based on the support +they receive from others, and the fact that you're even considering +contributing to the Requests project is *very* generous of you. -This document lays out guidelines and advice for contributing to Requests. -If you're thinking of contributing, start by reading this thoroughly and -getting a feel for how contributing to the project works. If you have any +This document lays out guidelines and advice for contributing to this project. +If you're thinking of contributing, please start by reading this document and +getting a feel for how contributing to this project works. If you have any questions, feel free to reach out to either `Ian Cordasco`_ or `Cory Benfield`_, the primary maintainers. +.. _Ian Cordasco: http://www.coglib.com/~icordasc/ +.. _Cory Benfield: https://lukasa.co.uk/about + +If you have non-technical feedback, philosophical ponderings, crazy ideas, or +other general thoughts about Requests or its position within the Python +ecosystem, the BDFL, `Kenneth Reitz`_, would love to hear from you. + The guide is split into sections based on the type of contribution you're thinking of making, with a section that covers general guidelines for all contributors. -.. _Ian Cordasco: http://www.coglib.com/~icordasc/ -.. _Cory Benfield: https://lukasa.co.uk/about - - -All Contributions ------------------ +.. _Kenneth Reitz: mailto:me@kennethreitz.org Be Cordial -~~~~~~~~~~ +---------- -**Be cordial or be on your way.** + **Be cordial or be on your way**. *—Kenneth Reitz* Requests has one very important rule governing all forms of contribution, including reporting bugs or requesting features. This golden rule is -`be cordial or be on your way`_. **All contributions are welcome**, as long as +"`be cordial or be on your way`_". + +**All contributions are welcome**, as long as everyone involved is treated with respect. .. _be cordial or be on your way: http://kennethreitz.org/be-cordial-or-be-on-your-way/ @@ -41,7 +44,7 @@ everyone involved is treated with respect. .. _early-feedback: Get Early Feedback -~~~~~~~~~~~~~~~~~~ +------------------ If you are contributing, do not feel the need to sit on your contribution until it is perfectly polished and complete. It helps everyone involved for you to @@ -51,22 +54,23 @@ getting that contribution accepted, and can save you from putting a lot of work into a contribution that is not suitable for the project. Contribution Suitability -~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------ -The project maintainer has the last word on whether or not a contribution is -suitable for Requests. All contributions will be considered, but from time -to time contributions will be rejected because they do not suit the project. +Our project maintainers have the last word on whether or not a contribution is +suitable for Requests. All contributions will be considered carefully, but from +time to time, contributions will be rejected because they do not suit the +current goals or needs of the project. -If your contribution is rejected, don't despair! So long as you followed these -guidelines, you'll have a much better chance of getting your next contribution -accepted. +If your contribution is rejected, don't despair! As long as you followed these +guidelines, you will have a much better chance of getting your next +contribution accepted. Code Contributions ------------------ -Steps -~~~~~ +Steps for Submitting Code +~~~~~~~~~~~~~~~~~~~~~~~~~ When contributing code, you'll want to follow this checklist: @@ -104,6 +108,56 @@ asking for help. Please also check the :ref:`early-feedback` section. +Kenneth Reitz's Code Style™ +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The Requests codebase uses the `PEP 8`_ code style. + +In addition to the standards outlined in PEP 8, we have a few guidelines: + +- Line-length can exceed 79 characters, to 100, when convenient. +- Line-length can exceed 100 characters, when doing otherwise would be *terribly* inconvenient. +- Always use single-quoted strings (e.g. ``'#flatearth'``), unless a single-quote occurs within the string. + +Additionally, one of the styles that PEP8 recommends for `line continuations`_ +completely lacks all sense of taste, and is not to be permitted within +the Requests codebase:: + + # Aligned with opening delimiter. + foo = long_function_name(var_one, var_two, + var_three, var_four) + +No. Just don't. Please. + +Docstrings are to follow the following syntaxes:: + + def the_earth_is_flat(): + """NASA divided up the seas into thirty-three degrees.""" + pass + +:: + + def fibonacci_spiral_tool(): + """With my feet upon the ground I lose myself / between the sounds + and open wide to suck it in. / I feel it move across my skin. / I'm + reaching up and reaching out. / I'm reaching for the random or + whatever will bewilder me. / Whatever will bewilder me. / And + following our will and wind we may just go where no one's been. / + We'll ride the spiral to the end and may just go where no one's + been. + + Spiral out. Keep going... + """ + pass + +All functions, methods, and classes are to contain docstrings. Object data +model methods (e.g. ``__repr__``) are typically the exception to this rule. + +Thanks for helping to make the world a better place! + +.. _PEP 8: http://pep8.org +.. _line continuations: https://www.python.org/dev/peps/pep-0008/#indentation + Documentation Contributions --------------------------- @@ -112,9 +166,12 @@ the ``docs/`` directory of the codebase. They're written in `reStructuredText`_, and use `Sphinx`_ to generate the full suite of documentation. -When contributing documentation, please attempt to follow the style of the +When contributing documentation, please do your best to follow the style of the documentation files. This means a soft-limit of 79 characters wide in your text -files and a semi-formal prose style. +files and a semi-formal, yet friendly and approachable, prose style. + +When presenting Python code, use single-quoted strings (``'hello'`` instead of +``"hello"``). .. _reStructuredText: http://docutils.sourceforge.net/rst.html .. _Sphinx: http://sphinx-doc.org/index.html @@ -136,10 +193,14 @@ of other contributors, and should be avoided as much as possible. Feature Requests ---------------- -Requests is in a perpetual feature freeze. The maintainers believe that -requests contains every major feature currently required by the vast majority -of users. +Requests is in a perpetual feature freeze, only the BDFL can add or approve of +new features. The maintainers believe that Requests is a feature-complete +piece of software at this time. + +One of the most important skills to have while maintaining a largely-used +open source project is learning the ability to say "no" to suggested changes, +while keeping an open ear and mind. If you believe there is a feature missing, feel free to raise a feature request, but please do be aware that the overwhelming likelihood is that your -feature request will not be accepted. +feature request will not be accepted. \ No newline at end of file diff --git a/docs/dev/philosophy.rst b/docs/dev/philosophy.rst index 8c9490ab..ab9f37ae 100644 --- a/docs/dev/philosophy.rst +++ b/docs/dev/philosophy.rst @@ -33,8 +33,6 @@ Requests has no *active* plans to be included in the standard library. This deci Essentially, the standard library is where a library goes to die. It is appropriate for a module to be included when active development is no longer necessary. -Requests just reached v1.0.0. This huge milestone marks a major step in the right direction. - Linux Distro Packages ~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/index.rst b/docs/index.rst index 617e01a3..5eb643e1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,18 +8,14 @@ Requests: HTTP for Humans Release v\ |version|. (:ref:`Installation `) -Requests is an :ref:`Apache2 Licensed ` HTTP library, written in -Python, for human beings. +Requests is the only *Non-GMO* HTTP library for Python, safe for human +consumption. -Python's standard **urllib2** module provides most of -the HTTP capabilities you need, but the API is thoroughly **broken**. -It was built for a different time — and a different web. It requires an -*enormous* amount of work (even method overrides) to perform the simplest of -tasks. +**Warning:** Recreational use of other HTTP libraries may result in dangerous side-effects, +including: security vulnerabilities, verbose code, reinventing the wheel, +constantly reading documentation, depression, headaches, or even death. -Things shouldn’t be this way. Not in Python. - -:: +Behold, the power of Requests:: >>> r = requests.get('https://api.github.com/user', auth=('user', 'pass')) >>> r.status_code @@ -33,22 +29,22 @@ Things shouldn’t be this way. Not in Python. >>> r.json() {u'private_gists': 419, u'total_private_repos': 77, ...} -See `similar code, without Requests `_. +See `similar code, sans Requests `_. -Requests takes all of the work out of Python HTTP/1.1 — making your integration -with web services seamless. There's no need to manually add query strings to -your URLs, or to form-encode your POST data. Keep-alive and HTTP connection -pooling are 100% automatic, powered by `urllib3 `_, + +Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, without the +need for manual labor. There's no need to manually add query strings to your +URLs, or to form-encode your POST data. Keep-alive and HTTP connection pooling +are 100% automatic, powered by `urllib3 `_, which is embedded within Requests. - -Testimonials ------------- +User Testimonials +----------------- Her Majesty's Government, Amazon, Google, Twilio, Runscope, Mozilla, Heroku, PayPal, NPR, Obama for America, Transifex, Native Instruments, The Washington -Post, Twitter, SoundCloud, Kippt, Readability, Sony, and Federal US Institutions that prefer to be unnamed -use Requests internally. It has been downloaded over 60,000,000 times from PyPI. +Post, Twitter, SoundCloud, Kippt, Readability, Sony, and Federal U.S. +Institutions that prefer to be unnamed claim to use Requests internally. **Armin Ronacher** Requests is the perfect example how beautiful an API can be with the @@ -66,9 +62,11 @@ use Requests internally. It has been downloaded over 60,000,000 times from PyPI. Python HTTP: When in doubt, or when not in doubt, use Requests. Beautiful, simple, Pythonic. +Requests is one of the most downloaded Python packages of all time, pulling in +over 7,000,000 downloads every month. All the cool kids are doing it! -Feature Support ---------------- +Supported Features +------------------ Requests is ready for today's web. @@ -79,16 +77,21 @@ Requests is ready for today's web. - Basic/Digest Authentication - Elegant Key/Value Cookies - Automatic Decompression +- Automatic Content Decoding - Unicode Response Bodies - Multipart File Uploads +- HTTP(S) Proxy Support - Connection Timeouts -- ``.netrc`` support -- Python 2.6—3.5 -- Thread-safe. +- Streaming Downloads +- ``.netrc`` Support +- Chunked Requests +- Thread-safety + +Requests supports Python 2.6 — 3.5, and runs great on PyPy. -User Guide ----------- +The User Guide +-------------- This part of the documentation, which is mostly prose, begins with some background information about Requests, then focuses on step-by-step @@ -104,8 +107,8 @@ instructions for getting the most out of Requests. user/authentication -Community Guide ------------------ +The Community Guide +------------------- This part of the documentation, which is mostly prose, details the Requests ecosystem and community. @@ -121,10 +124,10 @@ Requests ecosystem and community. community/updates community/release-process -API Documentation ------------------ +The API Documentation / Guide +----------------------------- -If you are looking for information on a specific function, class or method, +If you are looking for information on a specific function, class, or method, this part of the documentation is for you. .. toctree:: @@ -133,16 +136,19 @@ this part of the documentation is for you. api -Contributor Guide ------------------ +The Contributor Guide +--------------------- If you want to contribute to the project, this part of the documentation is for you. .. toctree:: - :maxdepth: 1 + :maxdepth: 3 dev/contributing dev/philosophy dev/todo dev/authors + +There are no more guides. You are now guideless. +Good luck. diff --git a/docs/make.bat b/docs/make.bat index 4441160c..9eaf9b88 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -1,190 +1,263 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Requests.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Requests.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -:end +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + echo. coverage to run coverage check of the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +REM Check if sphinx-build is available and fallback to Python version if any +%SPHINXBUILD% 1>NUL 2>NUL +if errorlevel 9009 goto sphinx_python +goto sphinx_ok + +:sphinx_python + +set SPHINXBUILD=python -m sphinx.__init__ +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +:sphinx_ok + + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Requests.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Requests.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "coverage" ( + %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage + if errorlevel 1 exit /b 1 + echo. + echo.Testing of coverage in the sources finished, look at the ^ +results in %BUILDDIR%/coverage/python.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index bf596036..00000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -alabaster==0.7.3 -Sphinx==1.1.3 diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 9c31950d..8264e85d 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -24,7 +24,7 @@ Let's persist some cookies across requests:: s = requests.Session() s.get('http://httpbin.org/cookies/set/sessioncookie/123456789') - r = s.get("http://httpbin.org/cookies") + r = s.get('http://httpbin.org/cookies') print(r.text) # '{"cookies": {"sessioncookie": "123456789"}}' @@ -50,6 +50,7 @@ requests, even if using a session. This example will only send the cookies with the first request, but not the second:: s = requests.Session() + r = s.get('http://httpbin.org/cookies', cookies={'from-my': 'browser'}) print(r.text) # '{"cookies": {"from-my": "browser"}}' @@ -129,14 +130,15 @@ request. The simple recipe for this is the following:: from requests import Request, Session s = Session() - req = Request('GET', url, - data=data, - headers=header - ) + + req = Request('POST', url, data=data, headers=headers) prepped = req.prepare() # do something with prepped.body + prepped.body = 'No, I want exactly this as the body.' + # do something with prepped.headers + del prepped.headers['Content-Type'] resp = s.send(prepped, stream=stream, @@ -165,15 +167,15 @@ applied, replace the call to :meth:`Request.prepare() from requests import Request, Session s = Session() - req = Request('GET', url, - data=data - headers=headers - ) + req = Request('GET', url, data=data, headers=headers) prepped = s.prepare_request(req) # do something with prepped.body + prepped.body = 'Seriously, send exactly these bytes.' + # do something with prepped.headers + prepped.headers['Keep-Dead'] = 'parrot' resp = s.send(prepped, stream=stream, @@ -190,21 +192,25 @@ applied, replace the call to :meth:`Request.prepare() SSL Cert Verification --------------------- -Requests can verify SSL certificates for HTTPS requests, just like a web browser. -To check a host's SSL certificate, you can use the ``verify`` argument:: +Requests verifies SSL certificates for HTTPS requests, just like a web browser. +By default, SSL verification is enabled, and requests will throw a SSLError if +it's unable to verify the certificate:: - >>> requests.get('https://kennethreitz.com', verify=True) - requests.exceptions.SSLError: hostname 'kennethreitz.com' doesn't match either of '*.herokuapp.com', 'herokuapp.com' + >>> requests.get('https://requestb.in') + requests.exceptions.SSLError: hostname 'requestb.in' doesn't match either of '*.herokuapp.com', 'herokuapp.com' -I don't have SSL setup on this domain, so it fails. Excellent. GitHub does though:: +I don't have SSL setup on this domain, so it throws an exception. Excellent. GitHub does though:: - >>> requests.get('https://github.com', verify=True) + >>> requests.get('https://github.com') You can pass ``verify`` the path to a CA_BUNDLE file or directory with certificates of trusted CAs:: >>> requests.get('https://github.com', verify='/path/to/certfile') +.. note:: If ``verify`` is set to a path to a directory, the directory must have been processed using + the c_rehash utility supplied with OpenSSL. + This list of trusted CAs can also be specified through the ``REQUESTS_CA_BUNDLE`` environment variable. Requests can also ignore verifying the SSL certificate if you set ``verify`` to False. @@ -223,7 +229,7 @@ file's path:: >>> requests.get('https://kennethreitz.com', cert=('/path/client.cert', '/path/client.key')) -If you specify a wrong path or an invalid cert:: +If you specify a wrong path or an invalid cert, you'll get a SSLError:: >>> requests.get('https://kennethreitz.com', cert='/wrong_path/client.pem') SSLError: [Errno 336265225] _ssl.c:347: error:140B0009:SSL routines:SSL_CTX_use_PrivateKey_file:PEM lib @@ -249,7 +255,7 @@ system. For the sake of security we recommend upgrading certifi frequently! .. _HTTP persistent connection: https://en.wikipedia.org/wiki/HTTP_persistent_connection -.. _connection pooling: https://urllib3.readthedocs.org/en/latest/pools.html +.. _connection pooling: https://urllib3.readthedocs.io/en/latest/pools.html .. _certifi: http://certifi.io/ .. _Mozilla trust store: https://hg.mozilla.org/mozilla-central/raw-file/tip/security/nss/lib/ckfw/builtins/certdata.txt @@ -356,15 +362,16 @@ POST Multiple Multipart-Encoded Files ------------------------------------- You can send multiple files in one request. For example, suppose you want to -upload image files to an HTML form with a multiple file field 'images': +upload image files to an HTML form with a multiple file field 'images':: -To do that, just set files to a list of tuples of (form_field_name, file_info): +To do that, just set files to a list of tuples of ``(form_field_name, file_info)``:: >>> url = 'http://httpbin.org/post' - >>> multiple_files = [('images', ('foo.png', open('foo.png', 'rb'), 'image/png')), - ('images', ('bar.png', open('bar.png', 'rb'), 'image/png'))] + >>> multiple_files = [ + ('images', ('foo.png', open('foo.png', 'rb'), 'image/png')), + ('images', ('bar.png', open('bar.png', 'rb'), 'image/png'))] >>> r = requests.post(url, files=multiple_files) >>> r.text { @@ -491,7 +498,9 @@ set ``stream`` to ``True`` and iterate over the response with lines = r.iter_lines() # Save the first line for later or just skip it + first_line = next(lines) + for line in lines: print(line) @@ -506,11 +515,11 @@ If you need to use a proxy, you can configure individual requests with the import requests proxies = { - "http": "http://10.10.1.10:3128", - "https": "http://10.10.1.10:1080", + 'http': 'http://10.10.1.10:3128', + 'https': 'http://10.10.1.10:1080', } - requests.get("http://example.org", proxies=proxies) + requests.get('http://example.org', proxies=proxies) You can also configure proxies by setting the environment variables ``HTTP_PROXY`` and ``HTTPS_PROXY``. @@ -519,15 +528,14 @@ You can also configure proxies by setting the environment variables $ export HTTP_PROXY="http://10.10.1.10:3128" $ export HTTPS_PROXY="http://10.10.1.10:1080" + $ python >>> import requests - >>> requests.get("http://example.org") + >>> requests.get('http://example.org') To use HTTP Basic Auth with your proxy, use the `http://user:password@host/` syntax:: - proxies = { - "http": "http://user:pass@10.10.1.10:3128/", - } + proxies = {'http': 'http://user:pass@10.10.1.10:3128/'} To give a proxy for a specific scheme and host, use the `scheme://hostname` form for the key. This will match for @@ -535,12 +543,33 @@ any request to the given scheme and exact hostname. :: - proxies = { - "http://10.20.1.128": "http://10.10.1.10:5323", - } + proxies = {'http://10.20.1.128': 'http://10.10.1.10:5323'} Note that proxy URLs must include the scheme. +SOCKS +^^^^^ + +.. versionadded:: 2.10.0 + +In addition to basic HTTP proxies, requests also supports proxies using the +SOCKS protocol. This is an optional feature that requires that additional +third-party libraries be installed before use. + +You can get the dependencies for this feature from ``pip``: + +.. code-block:: bash + + $ pip install requests[socks] + +Once you've installed those dependencies, using a SOCKS proxy is just as easy +as using a HTTP one:: + + proxies = { + 'http': 'socks5://user:pass@host:port', + 'https': 'socks5://user:pass@host:port' + } + .. _compliance: Compliance @@ -602,10 +631,13 @@ So, GitHub returns JSON. That's great, we can use the :meth:`r.json :: >>> commit_data = r.json() + >>> print(commit_data.keys()) [u'committer', u'author', u'url', u'tree', u'sha', u'parents', u'message'] + >>> print(commit_data[u'committer']) {u'date': u'2012-05-10T11:10:50-07:00', u'email': u'me@kennethreitz.com', u'name': u'Kenneth Reitz'} + >>> print(commit_data[u'message']) makin' history @@ -645,9 +677,12 @@ already exists, we will use it as an example. Let's start by getting it. >>> r = requests.get('https://api.github.com/repos/kennethreitz/requests/issues/482') >>> r.status_code 200 + >>> issue = json.loads(r.text) + >>> print(issue[u'title']) Feature any http verb in docs + >>> print(issue[u'comments']) 3 @@ -658,9 +693,12 @@ Cool, we have three comments. Let's take a look at the last of them. >>> r = requests.get(r.url + u'/comments') >>> r.status_code 200 + >>> comments = r.json() + >>> print(comments[0].keys()) [u'body', u'url', u'created_at', u'updated_at', u'user', u'id'] + >>> print(comments[2][u'body']) Probably in the "advanced" section @@ -680,6 +718,7 @@ is to POST to the thread. Let's do it. >>> body = json.dumps({u"body": u"Sounds great! I'll get right on it!"}) >>> url = u"https://api.github.com/repos/kennethreitz/requests/issues/482/comments" + >>> r = requests.post(url=url, data=body) >>> r.status_code 404 @@ -692,9 +731,11 @@ the very common Basic Auth. >>> from requests.auth import HTTPBasicAuth >>> auth = HTTPBasicAuth('fake@example.com', 'not_a_real_password') + >>> r = requests.post(url=url, data=body, auth=auth) >>> r.status_code 201 + >>> content = r.json() >>> print(content[u'body']) Sounds great! I'll get right on it. @@ -708,8 +749,10 @@ that. >>> print(content[u"id"]) 5804413 + >>> body = json.dumps({u"body": u"Sounds great! I'll get right on it once I feed my cat."}) >>> url = u"https://api.github.com/repos/kennethreitz/requests/issues/comments/5804413" + >>> r = requests.patch(url=url, data=body, auth=auth) >>> r.status_code 200 @@ -830,10 +873,9 @@ SSLv3: """"Transport adapter" that allows us to use SSLv3.""" def init_poolmanager(self, connections, maxsize, block=False): - self.poolmanager = PoolManager(num_pools=connections, - maxsize=maxsize, - block=block, - ssl_version=ssl.PROTOCOL_SSLv3) + self.poolmanager = PoolManager( + num_pools=connections, maxsize=maxsize, + block=block, ssl_version=ssl.PROTOCOL_SSLv3) .. _`described here`: http://www.kennethreitz.org/essays/the-future-of-python-http .. _`urllib3`: https://github.com/shazow/urllib3 diff --git a/docs/user/install.rst b/docs/user/install.rst index 5f0ef9c4..c3f0084e 100644 --- a/docs/user/install.rst +++ b/docs/user/install.rst @@ -7,23 +7,19 @@ This part of the documentation covers the installation of Requests. The first step to using any software package is getting it properly installed. -Distribute & Pip ----------------- +Pip Install Requests +-------------------- -Installing Requests is simple with `pip `_, just run -this in your terminal:: +To install Requests, simply run this simple command in your terminal of choice:: $ pip install requests -or, with `easy_install `_:: +If you don't have `pip `_ installed (tisk tisk!), +`this Python installation guide `_ +can guide you through the process. - $ easy_install requests - -But, you really `shouldn't do that `_. - - -Get the Code ------------- +Get the Source Code +------------------- Requests is actively developed on GitHub, where the code is `always available `_. @@ -32,16 +28,12 @@ You can either clone the public repository:: $ git clone git://github.com/kennethreitz/requests.git -Download the `tarball `_:: +Or, download the `tarball `_:: $ curl -OL https://github.com/kennethreitz/requests/tarball/master + # optionally, zipball is also available (for Windows users). -Or, download the `zipball `_:: - - $ curl -OL https://github.com/kennethreitz/requests/zipball/master - - -Once you have a copy of the source, you can embed it in your Python package, -or install it into your site-packages easily:: +Once you have a copy of the source, you can embed it in your own Python +package, or install it into your site-packages easily:: $ python setup.py install diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 10037fe6..afdabe26 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -37,15 +37,15 @@ get all the information we need from this object. Requests' simple API means that all forms of HTTP request are as obvious. For example, this is how you make an HTTP POST request:: - >>> r = requests.post("http://httpbin.org/post", data = {"key":"value"}) + >>> r = requests.post('http://httpbin.org/post', data = {'key':'value'}) Nice, right? What about the other HTTP request types: PUT, DELETE, HEAD and OPTIONS? These are all just as simple:: - >>> r = requests.put("http://httpbin.org/put", data = {"key":"value"}) - >>> r = requests.delete("http://httpbin.org/delete") - >>> r = requests.head("http://httpbin.org/get") - >>> r = requests.options("http://httpbin.org/get") + >>> r = requests.put('http://httpbin.org/put', data = {'key':'value'}) + >>> r = requests.delete('http://httpbin.org/delete') + >>> r = requests.head('http://httpbin.org/get') + >>> r = requests.options('http://httpbin.org/get') That's all well and good, but it's also only the start of what Requests can do. @@ -63,7 +63,7 @@ Requests allows you to provide these arguments as a dictionary, using the following code:: >>> payload = {'key1': 'value1', 'key2': 'value2'} - >>> r = requests.get("http://httpbin.org/get", params=payload) + >>> r = requests.get('http://httpbin.org/get', params=payload) You can see that the URL has been correctly encoded by printing the URL:: @@ -76,7 +76,8 @@ URL's query string. You can also pass a list of items as a value:: >>> payload = {'key1': 'value1', 'key2': ['value2', 'value3']} - >>> r = requests.get("http://httpbin.org/get", params=payload) + + >>> r = requests.get('http://httpbin.org/get', params=payload) >>> print(r.url) http://httpbin.org/get?key1=value1&key2=value2&key2=value3 @@ -87,6 +88,7 @@ We can read the content of the server's response. Consider the GitHub timeline again:: >>> import requests + >>> r = requests.get('https://api.github.com/events') >>> r.text u'[{"repository":{"open_issues":0,"url":"https://github.com/... @@ -131,6 +133,7 @@ use the following code:: >>> from PIL import Image >>> from StringIO import StringIO + >>> i = Image.open(StringIO(r.content)) @@ -140,6 +143,7 @@ JSON Response Content There's also a builtin JSON decoder, in case you're dealing with JSON data:: >>> import requests + >>> r = requests.get('https://api.github.com/events') >>> r.json() [{u'repository': {u'open_issues': 0, u'url': 'https://github.com/... @@ -163,8 +167,10 @@ server, you can access ``r.raw``. If you want to do this, make sure you set ``stream=True`` in your initial request. Once you do, you can do this:: >>> r = requests.get('https://api.github.com/events', stream=True) + >>> r.raw + >>> r.raw.read(10) '\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03' @@ -189,7 +195,6 @@ If you'd like to add HTTP headers to a request, simply pass in a ``dict`` to the For example, we didn't specify our user-agent in the previous example:: - >>> import json >>> url = 'https://api.github.com/some/endpoint' >>> headers = {'user-agent': 'my-app/0.0.1'} @@ -215,6 +220,7 @@ To do this, simply pass a dictionary to the ``data`` argument. Your dictionary of data will automatically be form-encoded when the request is made:: >>> payload = {'key1': 'value1', 'key2': 'value2'} + >>> r = requests.post("http://httpbin.org/post", data=payload) >>> print(r.text) { @@ -232,6 +238,7 @@ you pass in a ``string`` instead of a ``dict``, that data will be posted directl For example, the GitHub API v3 accepts JSON-Encoded POST/PATCH data:: >>> import json + >>> url = 'https://api.github.com/some/endpoint' >>> payload = {'some': 'data'} @@ -298,7 +305,7 @@ In the event you are posting a very large file as a ``multipart/form-data`` request, you may want to stream the request. By default, ``requests`` does not support this, but there is a separate package which does - ``requests-toolbelt``. You should read `the toolbelt's documentation -`_ for more details about how to use it. +`_ for more details about how to use it. For sending multiple files in one request refer to the :ref:`advanced ` section. @@ -426,10 +433,13 @@ response. For example, GitHub redirects all HTTP requests to HTTPS:: >>> r = requests.get('http://github.com') + >>> r.url 'https://github.com/' + >>> r.status_code 200 + >>> r.history [] @@ -438,16 +448,20 @@ If you're using GET, OPTIONS, POST, PUT, PATCH or DELETE, you can disable redirection handling with the ``allow_redirects`` parameter:: >>> r = requests.get('http://github.com', allow_redirects=False) + >>> r.status_code 301 + >>> r.history [] If you're using HEAD, you can enable redirection as well:: >>> r = requests.head('http://github.com', allow_redirects=True) + >>> r.url 'https://github.com/' + >>> r.history [] diff --git a/requests/__init__.py b/requests/__init__.py index b5b45595..eef1986a 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -55,6 +55,12 @@ try: except ImportError: pass +import warnings + +# urllib3's DependencyWarnings should be silenced. +from .packages.urllib3.exceptions import DependencyWarning +warnings.simplefilter('ignore', DependencyWarning) + from . import utils from .models import Request, Response, PreparedRequest from .api import request, get, head, post, patch, put, delete, options @@ -63,7 +69,7 @@ from .status_codes import codes from .exceptions import ( RequestException, Timeout, URLRequired, TooManyRedirects, HTTPError, ConnectionError, - FileModeWarning, + FileModeWarning, ConnectTimeout, ReadTimeout ) # Set default logging handler to avoid "No handler found" warnings. diff --git a/requests/adapters.py b/requests/adapters.py index 0fc3b2ee..ddede3eb 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -19,7 +19,7 @@ from .packages.urllib3.util.retry import Retry from .compat import urlparse, basestring from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers, prepend_scheme_if_needed, get_auth_from_url, urldefragauth, - select_proxy) + select_proxy, to_native_string) from .structures import CaseInsensitiveDict from .packages.urllib3.exceptions import ClosedPoolError from .packages.urllib3.exceptions import ConnectTimeoutError @@ -33,9 +33,15 @@ from .packages.urllib3.exceptions import SSLError as _SSLError from .packages.urllib3.exceptions import ResponseError from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError) + ProxyError, RetryError, InvalidSchema) from .auth import _basic_auth_str +try: + from .packages.urllib3.contrib.socks import SOCKSProxyManager +except ImportError: + def SOCKSProxyManager(*args, **kwargs): + raise InvalidSchema("Missing dependencies for SOCKS support.") + DEFAULT_POOLBLOCK = False DEFAULT_POOLSIZE = 10 DEFAULT_RETRIES = 0 @@ -48,10 +54,24 @@ class BaseAdapter(object): def __init__(self): super(BaseAdapter, self).__init__() - def send(self): + def send(self, request, stream=False, timeout=None, verify=True, + cert=None, proxies=None): + """Sends PreparedRequest object. Returns Response object. + + :param request: The :class:`PreparedRequest ` being sent. + :param stream: (optional) Whether to stream the request content. + :param timeout: (optional) How long to wait for the server to send + data before giving up, as a float, or a :ref:`(connect timeout, + read timeout) ` tuple. + :type timeout: float or tuple + :param verify: (optional) Whether to verify SSL certificates. + :param cert: (optional) Any user-provided SSL certificate to be trusted. + :param proxies: (optional) The proxies dictionary to apply to the request. + """ raise NotImplementedError def close(self): + """Cleans up adapter specific items.""" raise NotImplementedError @@ -149,9 +169,22 @@ class HTTPAdapter(BaseAdapter): :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager. :returns: ProxyManager """ - if not proxy in self.proxy_manager: + if proxy in self.proxy_manager: + manager = self.proxy_manager[proxy] + elif proxy.lower().startswith('socks'): + username, password = get_auth_from_url(proxy) + manager = self.proxy_manager[proxy] = SOCKSProxyManager( + proxy, + username=username, + password=password, + num_pools=self._pool_connections, + maxsize=self._pool_maxsize, + block=self._pool_block, + **proxy_kwargs + ) + else: proxy_headers = self.proxy_headers(proxy) - self.proxy_manager[proxy] = proxy_from_url( + manager = self.proxy_manager[proxy] = proxy_from_url( proxy, proxy_headers=proxy_headers, num_pools=self._pool_connections, @@ -159,7 +192,7 @@ class HTTPAdapter(BaseAdapter): block=self._pool_block, **proxy_kwargs) - return self.proxy_manager[proxy] + return manager def cert_verify(self, conn, url, verify, cert): """Verify a SSL certificate. This method should not be called from user @@ -264,10 +297,12 @@ class HTTPAdapter(BaseAdapter): def close(self): """Disposes of any internal state. - Currently, this just closes the PoolManager, which closes pooled - connections. + Currently, this closes the PoolManager and any active ProxyManager, + which closes any pooled connections. """ self.poolmanager.clear() + for proxy in self.proxy_manager.values(): + proxy.clear() def request_url(self, request, proxies): """Obtain the url to use when making the final request. @@ -284,10 +319,16 @@ class HTTPAdapter(BaseAdapter): """ proxy = select_proxy(request.url, proxies) scheme = urlparse(request.url).scheme - if proxy and scheme != 'https': + + is_proxied_http_request = (proxy and scheme != 'https') + using_socks_proxy = False + if proxy: + proxy_scheme = urlparse(proxy).scheme.lower() + using_socks_proxy = proxy_scheme.startswith('socks') + + url = request.path_url + if is_proxied_http_request and not using_socks_proxy: url = urldefragauth(request.url) - else: - url = request.path_url return url @@ -437,6 +478,9 @@ class HTTPAdapter(BaseAdapter): if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) + if isinstance(e.reason, _ProxyError): + raise ProxyError(e, request=request) + raise ConnectionError(e, request=request) except ClosedPoolError as e: diff --git a/requests/api.py b/requests/api.py index 498b210a..269a2566 100644 --- a/requests/api.py +++ b/requests/api.py @@ -25,7 +25,11 @@ def request(method, url, session=None, **kwargs): :param json: (optional) json data to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. - :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': ('filename', fileobj)}``) for multipart encoding upload. + :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload. + ``file-tuple`` can be a 2-tuple ``('filename', fileobj)``, 3-tuple ``('filename', fileobj, 'content_type')`` + or a 4-tuple ``('filename', fileobj, 'content_type', custom_headers)``, where ``'content-type'`` is a string + defining the content type of the given file and ``custom_headers`` a dict-like object containing additional headers + to add for the file. :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read diff --git a/requests/auth.py b/requests/auth.py index edf4c8dc..73f8e9da 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -93,6 +93,7 @@ class HTTPDigestAuth(AuthBase): qop = self._thread_local.chal.get('qop') algorithm = self._thread_local.chal.get('algorithm') opaque = self._thread_local.chal.get('opaque') + hash_utf8 = None if algorithm is None: _algorithm = 'MD5' diff --git a/requests/cookies.py b/requests/cookies.py index b85fd2b6..eee5168f 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -277,6 +277,12 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): dictionary[cookie.name] = cookie.value return dictionary + def __contains__(self, name): + try: + return super(RequestsCookieJar, self).__contains__(name) + except CookieConflictError: + return True + def __getitem__(self, name): """Dict-like __getitem__() for compatibility with client code. Throws exception if there are more than one cookie with name. In that case, diff --git a/requests/models.py b/requests/models.py index ffd5f971..2c9e608a 100644 --- a/requests/models.py +++ b/requests/models.py @@ -104,8 +104,10 @@ class RequestEncodingMixin(object): """Build the body for a multipart/form-data request. Will successfully encode files when passed as a dict or a list of - 2-tuples. Order is retained if data is a list of 2-tuples but arbitrary + tuples. Order is retained if data is a list of tuples but arbitrary if parameters are supplied as a dict. + The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype) + or 4-tuples (filename, fileobj, contentype, custom_headers). """ if (not files): @@ -423,8 +425,12 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): length = None if not data and json is not None: + # urllib3 requires a bytes-like body. Python 2's json.dumps + # provides this natively, but Python 3 gives a Unicode string. content_type = 'application/json' body = complexjson.dumps(json) + if not isinstance(body, bytes): + body = body.encode('utf-8') is_stream = all([ hasattr(data, '__iter__'), @@ -468,9 +474,11 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): def prepare_content_length(self, body): if hasattr(body, 'seek') and hasattr(body, 'tell'): + curr_pos = body.tell() body.seek(0, 2) - self.headers['Content-Length'] = builtin_str(body.tell()) - body.seek(0, 0) + end_pos = body.tell() + self.headers['Content-Length'] = builtin_str(max(0, end_pos - curr_pos)) + body.seek(curr_pos, 0) elif body is not None: l = super_len(body) if l: @@ -793,7 +801,7 @@ class Response(object): :param \*\*kwargs: Optional arguments that ``json.loads`` takes. """ - if not self.encoding and len(self.content) > 3: + if not self.encoding and self.content and len(self.content) > 3: # No encoding set. JSON RFC 4627 section 3 states we should expect # UTF-8, -16 or -32. Detect which one to use; If the detection or # decoding fails, fall back to `self.text` (using chardet to make diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index e43991a9..73668991 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -32,7 +32,7 @@ except ImportError: __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.13.1' +__version__ = '1.15.1' __all__ = ( 'HTTPConnectionPool', @@ -68,22 +68,25 @@ def add_stderr_logger(level=logging.DEBUG): handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) logger.addHandler(handler) logger.setLevel(level) - logger.debug('Added a stderr logging handler to logger: %s' % __name__) + logger.debug('Added a stderr logging handler to logger: %s', __name__) return handler # ... Clean up. del NullHandler +# All warning filters *must* be appended unless you're really certain that they +# shouldn't be: otherwise, it's very hard for users to use most Python +# mechanisms to silence them. # SecurityWarning's always go off by default. warnings.simplefilter('always', exceptions.SecurityWarning, append=True) # SubjectAltNameWarning's should go off once per host -warnings.simplefilter('default', exceptions.SubjectAltNameWarning) +warnings.simplefilter('default', exceptions.SubjectAltNameWarning, append=True) # InsecurePlatformWarning's don't vary between requests, so we keep it default. warnings.simplefilter('default', exceptions.InsecurePlatformWarning, append=True) # SNIMissingWarnings should go off only once. -warnings.simplefilter('default', exceptions.SNIMissingWarning) +warnings.simplefilter('default', exceptions.SNIMissingWarning, append=True) def disable_warnings(category=exceptions.HTTPWarning): diff --git a/requests/packages/urllib3/_collections.py b/requests/packages/urllib3/_collections.py index 67f3ce99..77cee017 100644 --- a/requests/packages/urllib3/_collections.py +++ b/requests/packages/urllib3/_collections.py @@ -134,7 +134,7 @@ class HTTPHeaderDict(MutableMapping): def __init__(self, headers=None, **kwargs): super(HTTPHeaderDict, self).__init__() - self._container = {} + self._container = OrderedDict() if headers is not None: if isinstance(headers, HTTPHeaderDict): self._copy_from(headers) diff --git a/requests/packages/urllib3/connection.py b/requests/packages/urllib3/connection.py index 1e4cd417..5ce00804 100644 --- a/requests/packages/urllib3/connection.py +++ b/requests/packages/urllib3/connection.py @@ -1,5 +1,6 @@ from __future__ import absolute_import import datetime +import logging import os import sys import socket @@ -38,7 +39,7 @@ from .exceptions import ( SubjectAltNameWarning, SystemTimeWarning, ) -from .packages.ssl_match_hostname import match_hostname +from .packages.ssl_match_hostname import match_hostname, CertificateError from .util.ssl_ import ( resolve_cert_reqs, @@ -50,6 +51,10 @@ from .util.ssl_ import ( from .util import connection +from ._collections import HTTPHeaderDict + +log = logging.getLogger(__name__) + port_by_scheme = { 'http': 80, 'https': 443, @@ -162,6 +167,38 @@ class HTTPConnection(_HTTPConnection, object): conn = self._new_conn() self._prepare_conn(conn) + def request_chunked(self, method, url, body=None, headers=None): + """ + Alternative to the common request method, which sends the + body with chunked encoding and not as one block + """ + headers = HTTPHeaderDict(headers if headers is not None else {}) + skip_accept_encoding = 'accept-encoding' in headers + self.putrequest(method, url, skip_accept_encoding=skip_accept_encoding) + for header, value in headers.items(): + self.putheader(header, value) + if 'transfer-encoding' not in headers: + self.putheader('Transfer-Encoding', 'chunked') + self.endheaders() + + if body is not None: + stringish_types = six.string_types + (six.binary_type,) + if isinstance(body, stringish_types): + body = (body,) + for chunk in body: + if not chunk: + continue + if not isinstance(chunk, six.binary_type): + chunk = chunk.encode('utf8') + len_str = hex(len(chunk))[2:] + self.send(len_str.encode('utf-8')) + self.send(b'\r\n') + self.send(chunk) + self.send(b'\r\n') + + # After the if clause, to always have a closed body + self.send(b'0\r\n\r\n') + class HTTPSConnection(HTTPConnection): default_port = port_by_scheme['https'] @@ -265,21 +302,26 @@ class VerifiedHTTPSConnection(HTTPSConnection): 'for details.)'.format(hostname)), SubjectAltNameWarning ) - - # In case the hostname is an IPv6 address, strip the square - # brackets from it before using it to validate. This is because - # a certificate with an IPv6 address in it won't have square - # brackets around that address. Sadly, match_hostname won't do this - # for us: it expects the plain host part without any extra work - # that might have been done to make it palatable to httplib. - asserted_hostname = self.assert_hostname or hostname - asserted_hostname = asserted_hostname.strip('[]') - match_hostname(cert, asserted_hostname) + _match_hostname(cert, self.assert_hostname or hostname) self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED or self.assert_fingerprint is not None) +def _match_hostname(cert, asserted_hostname): + try: + match_hostname(cert, asserted_hostname) + except CertificateError as e: + log.error( + 'Certificate did not match expected hostname: %s. ' + 'Certificate: %s', asserted_hostname, cert + ) + # Add cert to exception and reraise so client code can inspect + # the cert when catching the exception, if they want to + e._peer_cert = cert + raise + + if ssl: # Make a copy for testing. UnverifiedHTTPSConnection = HTTPSConnection diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 995b4167..3fcfb120 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -69,7 +69,13 @@ class ConnectionPool(object): if not host: raise LocationValueError("No host specified.") - self.host = host + # httplib doesn't like it when we include brackets in ipv6 addresses + # Specifically, if we include brackets but also pass the port then + # httplib crazily doubles up the square brackets on the Host header. + # Instead, we need to make sure we never pass ``None`` as the port. + # However, for backward compatibility reasons we can't actually + # *assert* that. + self.host = host.strip('[]') self.port = port def __str__(self): @@ -203,8 +209,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): Return a fresh :class:`HTTPConnection`. """ self.num_connections += 1 - log.info("Starting new HTTP connection (%d): %s" % - (self.num_connections, self.host)) + log.info("Starting new HTTP connection (%d): %s", + self.num_connections, self.host) conn = self.ConnectionCls(host=self.host, port=self.port, timeout=self.timeout.connect_timeout, @@ -239,7 +245,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # If this is a persistent connection, check if it got disconnected if conn and is_connection_dropped(conn): - log.info("Resetting dropped connection: %s" % self.host) + log.info("Resetting dropped connection: %s", self.host) conn.close() if getattr(conn, 'auto_open', 1) == 0: # This is a proxied connection that has been mutated by @@ -272,7 +278,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): except Full: # This should never happen if self.block == True log.warning( - "Connection pool is full, discarding connection: %s" % + "Connection pool is full, discarding connection: %s", self.host) # Connection never got put back into the pool, close it. @@ -318,7 +324,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python 2.6 raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) - def _make_request(self, conn, method, url, timeout=_Default, + def _make_request(self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw): """ Perform a request on a given urllib connection object taken from our @@ -350,7 +356,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # conn.request() calls httplib.*.request, not the method in # urllib3.request. It also calls makefile (recv) on the socket. - conn.request(method, url, **httplib_request_kw) + if chunked: + conn.request_chunked(method, url, **httplib_request_kw) + else: + conn.request(method, url, **httplib_request_kw) # Reset the timeout for the recv() on the socket read_timeout = timeout_obj.read_timeout @@ -382,9 +391,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # AppEngine doesn't have a version attr. http_version = getattr(conn, '_http_vsn_str', 'HTTP/?') - log.debug("\"%s %s %s\" %s %s" % (method, url, http_version, - httplib_response.status, - httplib_response.length)) + log.debug("\"%s %s %s\" %s %s", method, url, http_version, + httplib_response.status, httplib_response.length) try: assert_header_parsing(httplib_response.msg) @@ -435,7 +443,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): def urlopen(self, method, url, body=None, headers=None, retries=None, redirect=True, assert_same_host=True, timeout=_Default, - pool_timeout=None, release_conn=None, **response_kw): + pool_timeout=None, release_conn=None, chunked=False, + **response_kw): """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all @@ -512,6 +521,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): back into the pool. If None, it takes the value of ``response_kw.get('preload_content', True)``. + :param chunked: + If True, urllib3 will send the body using chunked transfer + encoding. Otherwise, urllib3 will send the body using the standard + content-length form. Defaults to False. + :param \**response_kw: Additional parameters are passed to :meth:`urllib3.response.HTTPResponse.from_httplib` @@ -542,6 +556,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # complains about UnboundLocalError. err = None + # Keep track of whether we cleanly exited the except block. This + # ensures we do proper cleanup in finally. + clean_exit = False + try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) @@ -556,13 +574,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Make the request on the httplib connection object. httplib_response = self._make_request(conn, method, url, timeout=timeout_obj, - body=body, headers=headers) + body=body, headers=headers, + chunked=chunked) # If we're going to release the connection in ``finally:``, then - # the request doesn't need to know about the connection. Otherwise + # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. - response_conn = not release_conn and conn + response_conn = conn if not release_conn else None # Import httplib's response into our own wrapper object response = HTTPResponse.from_httplib(httplib_response, @@ -570,10 +589,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): connection=response_conn, **response_kw) - # else: - # The connection will be put back into the pool when - # ``response.release_conn()`` is called (implicitly by - # ``response.read()``) + # Everything went great! + clean_exit = True except Empty: # Timed out by queue. @@ -583,22 +600,19 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Close the connection. If a connection is reused on which there # was a Certificate error, the next request will certainly raise # another Certificate error. - conn = conn and conn.close() - release_conn = True + clean_exit = False raise SSLError(e) except SSLError: # Treat SSLError separately from BaseSSLError to preserve # traceback. - conn = conn and conn.close() - release_conn = True + clean_exit = False raise except (TimeoutError, HTTPException, SocketError, ProtocolError) as e: # Discard the connection for these exceptions. It will be # be replaced during the next _get_conn() call. - conn = conn and conn.close() - release_conn = True + clean_exit = False if isinstance(e, (SocketError, NewConnectionError)) and self.proxy: e = ProxyError('Cannot connect to proxy.', e) @@ -613,6 +627,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): err = e finally: + if not clean_exit: + # We hit some kind of exception, handled or otherwise. We need + # to throw the connection away unless explicitly told not to. + # Close the connection, set the variable to None, and make sure + # we put the None back in the pool to avoid leaking it. + conn = conn and conn.close() + release_conn = True + if release_conn: # Put the connection back to be reused. If the connection is # expired then it will be None, which will get replaced with a @@ -622,7 +644,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if not conn: # Try again log.warning("Retrying (%r) after connection " - "broken by '%r': %s" % (retries, err, url)) + "broken by '%r': %s", retries, err, url) return self.urlopen(method, url, body, headers, retries, redirect, assert_same_host, timeout=timeout, pool_timeout=pool_timeout, @@ -644,7 +666,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): raise return response - log.info("Redirecting %s -> %s" % (url, redirect_location)) + log.info("Redirecting %s -> %s", url, redirect_location) return self.urlopen( method, redirect_location, body, headers, retries=retries, redirect=redirect, @@ -654,9 +676,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Check if we should retry the HTTP response. if retries.is_forced_retry(method, status_code=response.status): - retries = retries.increment(method, url, response=response, _pool=self) + try: + retries = retries.increment(method, url, response=response, _pool=self) + except MaxRetryError: + if retries.raise_on_status: + # Release the connection for this response, since we're not + # returning it to be released manually. + response.release_conn() + raise + return response retries.sleep() - log.info("Forced retry: %s" % url) + log.info("Forced retry: %s", url) return self.urlopen( method, url, body, headers, retries=retries, redirect=redirect, @@ -742,7 +772,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): except AttributeError: # Platform-specific: Python 2.6 set_tunnel = conn._set_tunnel - if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older + if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older set_tunnel(self.host, self.port) else: set_tunnel(self.host, self.port, self.proxy_headers) @@ -754,8 +784,8 @@ class HTTPSConnectionPool(HTTPConnectionPool): Return a fresh :class:`httplib.HTTPSConnection`. """ self.num_connections += 1 - log.info("Starting new HTTPS connection (%d): %s" - % (self.num_connections, self.host)) + log.info("Starting new HTTPS connection (%d): %s", + self.num_connections, self.host) if not self.ConnectionCls or self.ConnectionCls is DummyConnection: raise SSLError("Can't connect to HTTPS URL because the SSL " @@ -812,6 +842,7 @@ def connection_from_url(url, **kw): >>> r = conn.request('GET', '/') """ scheme, host, port = get_host(url) + port = port or port_by_scheme.get(scheme, 80) if scheme == 'https': return HTTPSConnectionPool(host, port=port, **kw) else: diff --git a/requests/packages/urllib3/contrib/appengine.py b/requests/packages/urllib3/contrib/appengine.py index 884cdb22..f4289c0f 100644 --- a/requests/packages/urllib3/contrib/appengine.py +++ b/requests/packages/urllib3/contrib/appengine.py @@ -144,7 +144,7 @@ class AppEngineManager(RequestMethods): if retries.is_forced_retry(method, status_code=http_response.status): retries = retries.increment( method, url, response=http_response, _pool=self) - log.info("Forced retry: %s" % url) + log.info("Forced retry: %s", url) retries.sleep() return self.urlopen( method, url, @@ -164,6 +164,14 @@ class AppEngineManager(RequestMethods): if content_encoding == 'deflate': del urlfetch_resp.headers['content-encoding'] + transfer_encoding = urlfetch_resp.headers.get('transfer-encoding') + # We have a full response's content, + # so let's make sure we don't report ourselves as chunked data. + if transfer_encoding == 'chunked': + encodings = transfer_encoding.split(",") + encodings.remove('chunked') + urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings) + return HTTPResponse( # In order for decoding to work, we must present the content as # a file-like object. @@ -177,7 +185,7 @@ class AppEngineManager(RequestMethods): if timeout is Timeout.DEFAULT_TIMEOUT: return 5 # 5s is the default timeout for URLFetch. if isinstance(timeout, Timeout): - if timeout.read is not timeout.connect: + if timeout._read is not timeout._connect: warnings.warn( "URLFetch does not support granular timeout settings, " "reverting to total timeout.", AppEnginePlatformWarning) diff --git a/requests/packages/urllib3/contrib/ntlmpool.py b/requests/packages/urllib3/contrib/ntlmpool.py index c136a238..11d0b5c3 100644 --- a/requests/packages/urllib3/contrib/ntlmpool.py +++ b/requests/packages/urllib3/contrib/ntlmpool.py @@ -43,8 +43,8 @@ class NTLMConnectionPool(HTTPSConnectionPool): # Performs the NTLM handshake that secures the connection. The socket # must be kept open while requests are performed. self.num_connections += 1 - log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s' % - (self.num_connections, self.host, self.authurl)) + log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s', + self.num_connections, self.host, self.authurl) headers = {} headers['Connection'] = 'Keep-Alive' @@ -56,13 +56,13 @@ class NTLMConnectionPool(HTTPSConnectionPool): # Send negotiation message headers[req_header] = ( 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)) - log.debug('Request headers: %s' % headers) + log.debug('Request headers: %s', headers) conn.request('GET', self.authurl, None, headers) res = conn.getresponse() reshdr = dict(res.getheaders()) - log.debug('Response status: %s %s' % (res.status, res.reason)) - log.debug('Response headers: %s' % reshdr) - log.debug('Response data: %s [...]' % res.read(100)) + log.debug('Response status: %s %s', res.status, res.reason) + log.debug('Response headers: %s', reshdr) + log.debug('Response data: %s [...]', res.read(100)) # Remove the reference to the socket, so that it can not be closed by # the response object (we want to keep the socket open) @@ -87,12 +87,12 @@ class NTLMConnectionPool(HTTPSConnectionPool): self.pw, NegotiateFlags) headers[req_header] = 'NTLM %s' % auth_msg - log.debug('Request headers: %s' % headers) + log.debug('Request headers: %s', headers) conn.request('GET', self.authurl, None, headers) res = conn.getresponse() - log.debug('Response status: %s %s' % (res.status, res.reason)) - log.debug('Response headers: %s' % dict(res.getheaders())) - log.debug('Response data: %s [...]' % res.read()[:100]) + log.debug('Response status: %s %s', res.status, res.reason) + log.debug('Response headers: %s', dict(res.getheaders())) + log.debug('Response data: %s [...]', res.read()[:100]) if res.status != 200: if res.status == 401: raise Exception('Server rejected request: wrong ' diff --git a/requests/packages/urllib3/contrib/pyopenssl.py b/requests/packages/urllib3/contrib/pyopenssl.py index 5996153a..ed3b9cc3 100644 --- a/requests/packages/urllib3/contrib/pyopenssl.py +++ b/requests/packages/urllib3/contrib/pyopenssl.py @@ -54,9 +54,17 @@ except SyntaxError as e: import OpenSSL.SSL from pyasn1.codec.der import decoder as der_decoder from pyasn1.type import univ, constraint -from socket import _fileobject, timeout, error as SocketError +from socket import timeout, error as SocketError + +try: # Platform-specific: Python 2 + from socket import _fileobject +except ImportError: # Platform-specific: Python 3 + _fileobject = None + from urllib3.packages.backports.makefile import backport_makefile + import ssl import select +import six from .. import connection from .. import util @@ -90,7 +98,7 @@ _openssl_verify = { OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, } -DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS +DEFAULT_SSL_CIPHER_LIST = util.ssl_.DEFAULT_CIPHERS.encode('ascii') # OpenSSL will only write 16K at a time SSL_WRITE_BLOCKSIZE = 16384 @@ -104,6 +112,7 @@ def inject_into_urllib3(): connection.ssl_wrap_socket = ssl_wrap_socket util.HAS_SNI = HAS_SNI + util.IS_PYOPENSSL = True def extract_from_urllib3(): @@ -111,6 +120,7 @@ def extract_from_urllib3(): connection.ssl_wrap_socket = orig_connection_ssl_wrap_socket util.HAS_SNI = orig_util_HAS_SNI + util.IS_PYOPENSSL = False # Note: This is a slightly bug-fixed version of same from ndg-httpsclient. @@ -135,7 +145,7 @@ def get_subj_alt_name(peer_cert): for i in range(peer_cert.get_extension_count()): ext = peer_cert.get_extension(i) ext_name = ext.get_short_name() - if ext_name != 'subjectAltName': + if ext_name != b'subjectAltName': continue # PyOpenSSL returns extension data in ASN.1 encoded form @@ -167,13 +177,17 @@ class WrappedSocket(object): self.socket = socket self.suppress_ragged_eofs = suppress_ragged_eofs self._makefile_refs = 0 + self._closed = False def fileno(self): return self.socket.fileno() - def makefile(self, mode, bufsize=-1): - self._makefile_refs += 1 - return _fileobject(self, mode, bufsize, close=True) + # Copy-pasted from Python 3.5 source code + def _decref_socketios(self): + if self._makefile_refs > 0: + self._makefile_refs -= 1 + if self._closed: + self.close() def recv(self, *args, **kwargs): try: @@ -182,7 +196,7 @@ class WrappedSocket(object): if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): return b'' else: - raise SocketError(e) + raise SocketError(str(e)) except OpenSSL.SSL.ZeroReturnError as e: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: return b'' @@ -198,6 +212,27 @@ class WrappedSocket(object): else: return data + def recv_into(self, *args, **kwargs): + try: + return self.connection.recv_into(*args, **kwargs) + except OpenSSL.SSL.SysCallError as e: + if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): + return 0 + else: + raise SocketError(str(e)) + except OpenSSL.SSL.ZeroReturnError as e: + if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: + return 0 + else: + raise + except OpenSSL.SSL.WantReadError: + rd, wd, ed = select.select( + [self.socket], [], [], self.socket.gettimeout()) + if not rd: + raise timeout('The read operation timed out') + else: + return self.recv_into(*args, **kwargs) + def settimeout(self, timeout): return self.socket.settimeout(timeout) @@ -225,6 +260,7 @@ class WrappedSocket(object): def close(self): if self._makefile_refs < 1: try: + self._closed = True return self.connection.close() except OpenSSL.SSL.Error: return @@ -262,6 +298,16 @@ class WrappedSocket(object): self._makefile_refs -= 1 +if _fileobject: # Platform-specific: Python 2 + def makefile(self, mode, bufsize=-1): + self._makefile_refs += 1 + return _fileobject(self, mode, bufsize, close=True) +else: # Platform-specific: Python 3 + makefile = backport_makefile + +WrappedSocket.makefile = makefile + + def _verify_callback(cnx, x509, err_no, err_depth, return_code): return err_no == 0 @@ -285,7 +331,7 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, else: ctx.set_default_verify_paths() - # Disable TLS compression to migitate CRIME attack (issue #309) + # Disable TLS compression to mitigate CRIME attack (issue #309) OP_NO_COMPRESSION = 0x20000 ctx.set_options(OP_NO_COMPRESSION) @@ -293,6 +339,8 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ctx.set_cipher_list(DEFAULT_SSL_CIPHER_LIST) cnx = OpenSSL.SSL.Connection(ctx, sock) + if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3 + server_hostname = server_hostname.encode('utf-8') cnx.set_tlsext_host_name(server_hostname) cnx.set_connect_state() while True: diff --git a/requests/packages/urllib3/contrib/socks.py b/requests/packages/urllib3/contrib/socks.py new file mode 100644 index 00000000..3748fee5 --- /dev/null +++ b/requests/packages/urllib3/contrib/socks.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +""" +SOCKS support for urllib3 +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This contrib module contains provisional support for SOCKS proxies from within +urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and +SOCKS5. To enable its functionality, either install PySocks or install this +module with the ``socks`` extra. + +Known Limitations: + +- Currently PySocks does not support contacting remote websites via literal + IPv6 addresses. Any such connection attempt will fail. +- Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any + such connection attempt will fail. +""" +from __future__ import absolute_import + +try: + import socks +except ImportError: + import warnings + from ..exceptions import DependencyWarning + + warnings.warn(( + 'SOCKS support in urllib3 requires the installation of optional ' + 'dependencies: specifically, PySocks. For more information, see ' + 'https://urllib3.readthedocs.org/en/latest/contrib.html#socks-proxies' + ), + DependencyWarning + ) + raise + +from socket import error as SocketError, timeout as SocketTimeout + +from ..connection import ( + HTTPConnection, HTTPSConnection +) +from ..connectionpool import ( + HTTPConnectionPool, HTTPSConnectionPool +) +from ..exceptions import ConnectTimeoutError, NewConnectionError +from ..poolmanager import PoolManager +from ..util.url import parse_url + +try: + import ssl +except ImportError: + ssl = None + + +class SOCKSConnection(HTTPConnection): + """ + A plain-text HTTP connection that connects via a SOCKS proxy. + """ + def __init__(self, *args, **kwargs): + self._socks_options = kwargs.pop('_socks_options') + super(SOCKSConnection, self).__init__(*args, **kwargs) + + def _new_conn(self): + """ + Establish a new connection via the SOCKS proxy. + """ + extra_kw = {} + if self.source_address: + extra_kw['source_address'] = self.source_address + + if self.socket_options: + extra_kw['socket_options'] = self.socket_options + + try: + conn = socks.create_connection( + (self.host, self.port), + proxy_type=self._socks_options['socks_version'], + proxy_addr=self._socks_options['proxy_host'], + proxy_port=self._socks_options['proxy_port'], + proxy_username=self._socks_options['username'], + proxy_password=self._socks_options['password'], + timeout=self.timeout, + **extra_kw + ) + + except SocketTimeout as e: + raise ConnectTimeoutError( + self, "Connection to %s timed out. (connect timeout=%s)" % + (self.host, self.timeout)) + + except socks.ProxyError as e: + # This is fragile as hell, but it seems to be the only way to raise + # useful errors here. + if e.socket_err: + error = e.socket_err + if isinstance(error, SocketTimeout): + raise ConnectTimeoutError( + self, + "Connection to %s timed out. (connect timeout=%s)" % + (self.host, self.timeout) + ) + else: + raise NewConnectionError( + self, + "Failed to establish a new connection: %s" % error + ) + else: + raise NewConnectionError( + self, + "Failed to establish a new connection: %s" % e + ) + + except SocketError as e: # Defensive: PySocks should catch all these. + raise NewConnectionError( + self, "Failed to establish a new connection: %s" % e) + + return conn + + +# We don't need to duplicate the Verified/Unverified distinction from +# urllib3/connection.py here because the HTTPSConnection will already have been +# correctly set to either the Verified or Unverified form by that module. This +# means the SOCKSHTTPSConnection will automatically be the correct type. +class SOCKSHTTPSConnection(SOCKSConnection, HTTPSConnection): + pass + + +class SOCKSHTTPConnectionPool(HTTPConnectionPool): + ConnectionCls = SOCKSConnection + + +class SOCKSHTTPSConnectionPool(HTTPSConnectionPool): + ConnectionCls = SOCKSHTTPSConnection + + +class SOCKSProxyManager(PoolManager): + """ + A version of the urllib3 ProxyManager that routes connections via the + defined SOCKS proxy. + """ + pool_classes_by_scheme = { + 'http': SOCKSHTTPConnectionPool, + 'https': SOCKSHTTPSConnectionPool, + } + + def __init__(self, proxy_url, username=None, password=None, + num_pools=10, headers=None, **connection_pool_kw): + parsed = parse_url(proxy_url) + + if parsed.scheme == 'socks5': + socks_version = socks.PROXY_TYPE_SOCKS5 + elif parsed.scheme == 'socks4': + socks_version = socks.PROXY_TYPE_SOCKS4 + else: + raise ValueError( + "Unable to determine SOCKS version from %s" % proxy_url + ) + + self.proxy_url = proxy_url + + socks_options = { + 'socks_version': socks_version, + 'proxy_host': parsed.host, + 'proxy_port': parsed.port, + 'username': username, + 'password': password, + } + connection_pool_kw['_socks_options'] = socks_options + + super(SOCKSProxyManager, self).__init__( + num_pools, headers, **connection_pool_kw + ) + + self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py index 8e07eb61..f2e65917 100644 --- a/requests/packages/urllib3/exceptions.py +++ b/requests/packages/urllib3/exceptions.py @@ -180,6 +180,14 @@ class SNIMissingWarning(HTTPWarning): pass +class DependencyWarning(HTTPWarning): + """ + Warned when an attempt is made to import a module with missing optional + dependencies. + """ + pass + + class ResponseNotChunked(ProtocolError, ValueError): "Response needs to be chunked in order to read it as chunks." pass diff --git a/requests/packages/urllib3/fields.py b/requests/packages/urllib3/fields.py index c7d48113..8fa2a127 100644 --- a/requests/packages/urllib3/fields.py +++ b/requests/packages/urllib3/fields.py @@ -36,11 +36,11 @@ def format_header_param(name, value): result = '%s="%s"' % (name, value) try: result.encode('ascii') - except UnicodeEncodeError: + except (UnicodeEncodeError, UnicodeDecodeError): pass else: return result - if not six.PY3: # Python 2: + if not six.PY3 and isinstance(value, six.text_type): # Python 2: value = value.encode('utf-8') value = email.utils.encode_rfc2231(value, 'utf-8') value = '%s*=%s' % (name, value) diff --git a/requests/packages/urllib3/packages/backports/__init__.py b/requests/packages/urllib3/packages/backports/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/requests/packages/urllib3/packages/backports/makefile.py b/requests/packages/urllib3/packages/backports/makefile.py new file mode 100644 index 00000000..75b80dcf --- /dev/null +++ b/requests/packages/urllib3/packages/backports/makefile.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +""" +backports.makefile +~~~~~~~~~~~~~~~~~~ + +Backports the Python 3 ``socket.makefile`` method for use with anything that +wants to create a "fake" socket object. +""" +import io + +from socket import SocketIO + + +def backport_makefile(self, mode="r", buffering=None, encoding=None, + errors=None, newline=None): + """ + Backport of ``socket.makefile`` from Python 3.5. + """ + if not set(mode) <= set(["r", "w", "b"]): + raise ValueError( + "invalid mode %r (only r, w, b allowed)" % (mode,) + ) + writing = "w" in mode + reading = "r" in mode or not writing + assert reading or writing + binary = "b" in mode + rawmode = "" + if reading: + rawmode += "r" + if writing: + rawmode += "w" + raw = SocketIO(self, rawmode) + self._makefile_refs += 1 + if buffering is None: + buffering = -1 + if buffering < 0: + buffering = io.DEFAULT_BUFFER_SIZE + if buffering == 0: + if not binary: + raise ValueError("unbuffered streams must be binary") + return raw + if reading and writing: + buffer = io.BufferedRWPair(raw, raw, buffering) + elif reading: + buffer = io.BufferedReader(raw, buffering) + else: + assert writing + buffer = io.BufferedWriter(raw, buffering) + if binary: + return buffer + text = io.TextIOWrapper(buffer, encoding, errors, newline) + text.mode = mode + return text diff --git a/requests/packages/urllib3/packages/ssl_match_hostname/.gitignore b/requests/packages/urllib3/packages/ssl_match_hostname/.gitignore deleted file mode 100644 index 0a764a4d..00000000 --- a/requests/packages/urllib3/packages/ssl_match_hostname/.gitignore +++ /dev/null @@ -1 +0,0 @@ -env diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index f13e673d..1023dcba 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -18,16 +18,16 @@ from .util.retry import Retry __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] -pool_classes_by_scheme = { - 'http': HTTPConnectionPool, - 'https': HTTPSConnectionPool, -} - log = logging.getLogger(__name__) SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', 'ssl_version', 'ca_cert_dir') +pool_classes_by_scheme = { + 'http': HTTPConnectionPool, + 'https': HTTPSConnectionPool, +} + class PoolManager(RequestMethods): """ @@ -65,6 +65,9 @@ class PoolManager(RequestMethods): self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close()) + # Locally set the pool classes so other PoolManagers can override them. + self.pool_classes_by_scheme = pool_classes_by_scheme + def __enter__(self): return self @@ -81,7 +84,7 @@ class PoolManager(RequestMethods): by :meth:`connection_from_url` and companion methods. It is intended to be overridden for customization. """ - pool_cls = pool_classes_by_scheme[scheme] + pool_cls = self.pool_classes_by_scheme[scheme] kwargs = self.connection_pool_kw if scheme == 'http': kwargs = self.connection_pool_kw.copy() @@ -186,7 +189,7 @@ class PoolManager(RequestMethods): kw['retries'] = retries kw['redirect'] = redirect - log.info("Redirecting %s -> %s" % (url, redirect_location)) + log.info("Redirecting %s -> %s", url, redirect_location) return self.urlopen(method, redirect_location, **kw) diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 8f2a1b5c..ac1b2f19 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -221,6 +221,8 @@ class HTTPResponse(io.IOBase): On exit, release the connection back to the pool. """ + clean_exit = False + try: try: yield @@ -243,20 +245,27 @@ class HTTPResponse(io.IOBase): # This includes IncompleteRead. raise ProtocolError('Connection broken: %r' % e, e) - except Exception: - # The response may not be closed but we're not going to use it anymore - # so close it now to ensure that the connection is released back to the pool. - if self._original_response and not self._original_response.isclosed(): - self._original_response.close() - - # Closing the response may not actually be sufficient to close - # everything, so if we have a hold of the connection close that - # too. - if self._connection is not None: - self._connection.close() - - raise + # If no exception is thrown, we should avoid cleaning up + # unnecessarily. + clean_exit = True finally: + # If we didn't terminate cleanly, we need to throw away our + # connection. + if not clean_exit: + # The response may not be closed but we're not going to use it + # anymore so close it now to ensure that the connection is + # released back to the pool. + if self._original_response: + self._original_response.close() + + # Closing the response may not actually be sufficient to close + # everything, so if we have a hold of the connection close that + # too. + if self._connection: + self._connection.close() + + # If we hold the original response but it's closed now, we should + # return the connection back to the pool. if self._original_response and self._original_response.isclosed(): self.release_conn() @@ -387,6 +396,9 @@ class HTTPResponse(io.IOBase): if not self.closed: self._fp.close() + if self._connection: + self._connection.close() + @property def closed(self): if self._fp is None: diff --git a/requests/packages/urllib3/util/__init__.py b/requests/packages/urllib3/util/__init__.py index c6c6243c..4778cf99 100644 --- a/requests/packages/urllib3/util/__init__.py +++ b/requests/packages/urllib3/util/__init__.py @@ -6,6 +6,7 @@ from .response import is_fp_closed from .ssl_ import ( SSLContext, HAS_SNI, + IS_PYOPENSSL, assert_fingerprint, resolve_cert_reqs, resolve_ssl_version, @@ -26,6 +27,7 @@ from .url import ( __all__ = ( 'HAS_SNI', + 'IS_PYOPENSSL', 'SSLContext', 'Retry', 'Timeout', diff --git a/requests/packages/urllib3/util/response.py b/requests/packages/urllib3/util/response.py index bc723272..0b5c75c1 100644 --- a/requests/packages/urllib3/util/response.py +++ b/requests/packages/urllib3/util/response.py @@ -61,7 +61,7 @@ def assert_header_parsing(headers): def is_response_to_head(response): """ - Checks, wether a the request of a response has been a HEAD-request. + Checks whether the request of a response has been a HEAD-request. Handles the quirks of AppEngine. :param conn: diff --git a/requests/packages/urllib3/util/retry.py b/requests/packages/urllib3/util/retry.py index 03a01249..2d3aa20d 100644 --- a/requests/packages/urllib3/util/retry.py +++ b/requests/packages/urllib3/util/retry.py @@ -102,6 +102,11 @@ class Retry(object): :param bool raise_on_redirect: Whether, if the number of redirects is exhausted, to raise a MaxRetryError, or to return a response with a response code in the 3xx range. + + :param bool raise_on_status: Similar meaning to ``raise_on_redirect``: + whether we should raise an exception, or return a response, + if status falls in ``status_forcelist`` range and retries have + been exhausted. """ DEFAULT_METHOD_WHITELIST = frozenset([ @@ -112,7 +117,8 @@ class Retry(object): def __init__(self, total=10, connect=None, read=None, redirect=None, method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None, - backoff_factor=0, raise_on_redirect=True, _observed_errors=0): + backoff_factor=0, raise_on_redirect=True, raise_on_status=True, + _observed_errors=0): self.total = total self.connect = connect @@ -127,6 +133,7 @@ class Retry(object): self.method_whitelist = method_whitelist self.backoff_factor = backoff_factor self.raise_on_redirect = raise_on_redirect + self.raise_on_status = raise_on_status self._observed_errors = _observed_errors # TODO: use .history instead? def new(self, **kw): @@ -137,6 +144,7 @@ class Retry(object): status_forcelist=self.status_forcelist, backoff_factor=self.backoff_factor, raise_on_redirect=self.raise_on_redirect, + raise_on_status=self.raise_on_status, _observed_errors=self._observed_errors, ) params.update(kw) @@ -153,7 +161,7 @@ class Retry(object): redirect = bool(redirect) and None new_retries = cls(retries, redirect=redirect) - log.debug("Converted retries value: %r -> %r" % (retries, new_retries)) + log.debug("Converted retries value: %r -> %r", retries, new_retries) return new_retries def get_backoff_time(self): @@ -272,7 +280,7 @@ class Retry(object): if new_retry.is_exhausted(): raise MaxRetryError(_pool, url, error or ResponseError(cause)) - log.debug("Incremented Retry for (url='%s'): %r" % (url, new_retry)) + log.debug("Incremented Retry for (url='%s'): %r", url, new_retry) return new_retry diff --git a/requests/packages/urllib3/util/ssl_.py b/requests/packages/urllib3/util/ssl_.py index 67f83441..e8d9e7d2 100644 --- a/requests/packages/urllib3/util/ssl_.py +++ b/requests/packages/urllib3/util/ssl_.py @@ -12,6 +12,7 @@ from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning SSLContext = None HAS_SNI = False create_default_context = None +IS_PYOPENSSL = False # Maps the length of a digest to a possible hash function producing this digest HASHFUNC_MAP = { @@ -110,11 +111,12 @@ except ImportError: ) self.ciphers = cipher_suite - def wrap_socket(self, socket, server_hostname=None): + def wrap_socket(self, socket, server_hostname=None, server_side=False): warnings.warn( 'A true SSLContext object is not available. This prevents ' 'urllib3 from configuring SSL appropriately and may cause ' - 'certain SSL connections to fail. For more information, see ' + 'certain SSL connections to fail. You can upgrade to a newer ' + 'version of Python to solve this. For more information, see ' 'https://urllib3.readthedocs.org/en/latest/security.html' '#insecureplatformwarning.', InsecurePlatformWarning @@ -125,6 +127,7 @@ except ImportError: 'ca_certs': self.ca_certs, 'cert_reqs': self.verify_mode, 'ssl_version': self.protocol, + 'server_side': server_side, } if self.supports_set_ciphers: # Platform-specific: Python 2.7+ return wrap_socket(socket, ciphers=self.ciphers, **kwargs) @@ -308,8 +311,8 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, 'An HTTPS request has been made, but the SNI (Subject Name ' 'Indication) extension to TLS is not available on this platform. ' 'This may cause the server to present an incorrect TLS ' - 'certificate, which can cause validation failures. For more ' - 'information, see ' + 'certificate, which can cause validation failures. You can upgrade to ' + 'a newer version of Python to solve this. For more information, see ' 'https://urllib3.readthedocs.org/en/latest/security.html' '#snimissingwarning.', SNIMissingWarning diff --git a/requests/sessions.py b/requests/sessions.py index dc993d7f..f1f5522d 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -152,21 +152,7 @@ class SessionRedirectMixin(object): if response.is_permanent_redirect and request.url != prepared_request.url: self.redirect_cache[request.url] = prepared_request.url - # http://tools.ietf.org/html/rfc7231#section-6.4.4 - if (response.status_code == codes.see_other and method != 'HEAD'): - method = 'GET' - - # Do what the browsers do, despite standards... - # First, turn 302s into GETs. - if response.status_code == codes.found and method != 'HEAD': - method = 'GET' - - # Second, if a POST is responded to with a 301, turn it into a GET. - # This bizarre behaviour is explained in Issue 1704. - if response.status_code == codes.moved and method == 'POST': - method = 'GET' - - prepared_request.method = method + self.rebuild_method(prepared_request, resp) # https://github.com/kennethreitz/requests/issues/1084 if response.status_code not in (codes.temporary_redirect, codes.permanent_redirect): @@ -253,10 +239,10 @@ class SessionRedirectMixin(object): if self.trust_env and not should_bypass_proxies(url): environ_proxies = get_environ_proxies(url) - proxy = environ_proxies.get(scheme) + proxy = environ_proxies.get('all', environ_proxies.get(scheme)) if proxy: - new_proxies.setdefault(scheme, environ_proxies[scheme]) + new_proxies.setdefault(scheme, proxy) if 'Proxy-Authorization' in headers: del headers['Proxy-Authorization'] @@ -271,6 +257,28 @@ class SessionRedirectMixin(object): return new_proxies + def rebuild_method(self, prepared_request, response): + """When being redirected we may want to change the method of the request + based on certain specs or browser behavior. + """ + method = prepared_request.method + + # http://tools.ietf.org/html/rfc7231#section-6.4.4 + if response.status_code == codes.see_other and method != 'HEAD': + method = 'GET' + + # Do what the browsers do, despite standards... + # First, turn 302s into GETs. + if response.status_code == codes.found and method != 'HEAD': + method = 'GET' + + # Second, if a POST is responded to with a 301, turn it into a GET. + # This bizarre behaviour is explained in Issue 1704. + if response.status_code == codes.moved and method == 'POST': + method = 'GET' + + prepared_request.method = method + class Session(SessionRedirectMixin): """A Requests session. @@ -332,6 +340,8 @@ class Session(SessionRedirectMixin): #: Maximum number of redirects allowed. If the request exceeds this #: limit, a :class:`TooManyRedirects` exception is raised. + #: This defaults to requests.models.DEFAULT_REDIRECT_LIMIT, which is + #: 30. self.max_redirects = DEFAULT_REDIRECT_LIMIT #: Trust environment settings for proxy configuration, default @@ -446,6 +456,7 @@ class Session(SessionRedirectMixin): A CA_BUNDLE path can also be provided. Defaults to ``True``. :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. + :rtype: requests.Response """ # Create the Request. req = Request( @@ -559,24 +570,26 @@ class Session(SessionRedirectMixin): # It's possible that users might accidentally send a Request object. # Guard against that specific failure case. - if not isinstance(request, PreparedRequest): + if isinstance(request, Request): raise ValueError('You can only send PreparedRequests.') - # Automatically skip a redirect chain if we've already followed it before. - checked_urls = set() - while request.url in self.redirect_cache: - checked_urls.add(request.url) - new_url = self.redirect_cache.get(request.url) - if new_url in checked_urls: - break - request.url = new_url - - # Set-up variables for resolve_redirects and dispatching of hooks. + # Set up variables needed for resolve_redirects and dispatching of + # hooks allow_redirects = kwargs.pop('allow_redirects', True) stream = kwargs.get('stream') hooks = request.hooks - # Get the appropriate adapter to use. + # Resolve URL in redirect cache, if available. + if allow_redirects: + checked_urls = set() + while request.url in self.redirect_cache: + checked_urls.add(request.url) + new_url = self.redirect_cache.get(request.url) + if new_url in checked_urls: + break + request.url = new_url + + # Get the appropriate adapter to use adapter = self.get_adapter(url=request.url) # Start time (approximately) of the request. diff --git a/requests/status_codes.py b/requests/status_codes.py index a852574a..0137c91d 100644 --- a/requests/status_codes.py +++ b/requests/status_codes.py @@ -53,6 +53,7 @@ _codes = { 416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'), 417: ('expectation_failed',), 418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'), + 421: ('misdirected_request',), 422: ('unprocessable_entity', 'unprocessable'), 423: ('locked',), 424: ('failed_dependency', 'dependency'), diff --git a/requests/structures.py b/requests/structures.py index 3e5f2faa..991056e4 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -10,6 +10,8 @@ Data structures that power Requests. import collections +from .compat import OrderedDict + class CaseInsensitiveDict(collections.MutableMapping): """ @@ -40,7 +42,7 @@ class CaseInsensitiveDict(collections.MutableMapping): """ def __init__(self, data=None, **kwargs): - self._store = dict() + self._store = OrderedDict() if data is None: data = {} self.update(data, **kwargs) diff --git a/requests/utils.py b/requests/utils.py index 5c18e184..f9629112 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -14,9 +14,7 @@ import codecs import collections import io import os -import platform import re -import sys import socket import struct import warnings @@ -83,7 +81,14 @@ def super_len(o): ) if hasattr(o, 'tell'): - current_position = o.tell() + try: + current_position = o.tell() + except (OSError, IOError): + # This can happen in some weird situations, such as when the file + # is actually a special file descriptor like stdin. In this + # instance, we don't know what the length is, so set it to zero and + # let requests chunk it instead. + current_position = total_length return max(0, total_length - current_position) @@ -553,6 +558,10 @@ def should_bypass_proxies(url): if is_valid_cidr(proxy_ip): if address_in_network(ip, proxy_ip): return True + elif ip == proxy_ip: + # If no_proxy ip was defined in plain IP notation instead of cidr notation & + # matches the IP of the index + return True else: for host in no_proxy: if netloc.endswith(host) or netloc.split(':')[0].endswith(host): @@ -576,6 +585,7 @@ def should_bypass_proxies(url): return False + def get_environ_proxies(url): """Return a dict of environment proxies.""" if should_bypass_proxies(url): @@ -583,6 +593,7 @@ def get_environ_proxies(url): else: return getproxies() + def select_proxy(url, proxies): """Select a proxy for the url, if applicable. @@ -591,11 +602,24 @@ def select_proxy(url, proxies): """ proxies = proxies or {} urlparts = urlparse(url) - proxy = proxies.get(urlparts.scheme+'://'+urlparts.hostname) - if proxy is None: - proxy = proxies.get(urlparts.scheme) + if urlparts.hostname is None: + return proxies.get('all', proxies.get(urlparts.scheme)) + + proxy_keys = [ + 'all://' + urlparts.hostname, + 'all', + urlparts.scheme + '://' + urlparts.hostname, + urlparts.scheme, + ] + proxy = None + for proxy_key in proxy_keys: + if proxy_key in proxies: + proxy = proxies[proxy_key] + break + return proxy + def default_user_agent(name="python-requests"): """Return a string representing the default user agent.""" return '%s/%s' % (name, __version__) @@ -619,21 +643,19 @@ def parse_header_links(value): links = [] - replace_chars = " '\"" + replace_chars = ' \'"' - for val in re.split(", *<", value): + for val in re.split(', *<', value): try: - url, params = val.split(";", 1) + url, params = val.split(';', 1) except ValueError: url, params = val, '' - link = {} + link = {'url': url.strip('<> \'"')} - link["url"] = url.strip("<> '\"") - - for param in params.split(";"): + for param in params.split(';'): try: - key, value = param.split("=") + key, value = param.split('=') except ValueError: break @@ -680,8 +702,8 @@ def guess_json_utf(data): def prepend_scheme_if_needed(url, new_scheme): - '''Given a URL that may or may not have a scheme, prepend the given scheme. - Does not replace a present scheme with the one provided as an argument.''' + """Given a URL that may or may not have a scheme, prepend the given scheme. + Does not replace a present scheme with the one provided as an argument.""" scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme) # urlparse is a finicky beast, and sometimes decides that there isn't a @@ -712,8 +734,6 @@ def to_native_string(string, encoding='ascii'): string in the native string type, encoding and decoding where necessary. This assumes ASCII unless told otherwise. """ - out = None - if isinstance(string, builtin_str): out = string else: diff --git a/requirements-to-freeze.txt b/requirements-to-freeze.txt new file mode 100644 index 00000000..e8b9e354 --- /dev/null +++ b/requirements-to-freeze.txt @@ -0,0 +1,4 @@ +pytest +pytest-cov +pytest-httpbin +sphinx \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ad5da761..8426eecb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,24 @@ -py==1.4.30 -pytest==2.8.1 -pytest-cov==2.1.0 -pytest-httpbin==0.0.7 -httpbin==0.4.0 -wheel +alabaster==0.7.7 +Babel==2.2.0 +coverage==4.0.3 +decorator==4.0.9 +docutils==0.12 +Flask==0.10.1 +httpbin==0.4.1 +itsdangerous==0.24 +Jinja2==2.8 +MarkupSafe==0.23 +py==1.4.31 +Pygments==2.1.1 +PySocks==1.5.6 +pytest==2.8.7 +pytest-cov==2.2.1 +pytest-httpbin==0.2.0 +pytest-mock==0.11.0 +pytz==2015.7 +six==1.10.0 +snowballstemmer==1.2.1 +Sphinx==1.3.5 +sphinx-rtd-theme==0.1.9 +Werkzeug==0.11.4 +wheel==0.29.0 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 5e409001..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[wheel] -universal = 1 diff --git a/setup.py b/setup.py index 88b8e087..3a390522 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import os -import platform import re import sys @@ -94,5 +93,6 @@ setup( tests_require=test_requirements, extras_require={ 'security': ['pyOpenSSL>=0.13', 'ndg-httpsclient', 'pyasn1'], + 'socks': ['PySocks>=1.5.6'], }, ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..57d631c3 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# coding: utf-8 diff --git a/tests/compat.py b/tests/compat.py new file mode 100644 index 00000000..a26bd9f4 --- /dev/null +++ b/tests/compat.py @@ -0,0 +1,20 @@ +# coding: utf-8 +from requests.compat import is_py3 + + +try: + import StringIO +except ImportError: + import io as StringIO + +try: + from cStringIO import StringIO as cStringIO +except ImportError: + cStringIO = None + +if is_py3: + def u(s): + return s +else: + def u(s): + return s.decode('unicode-escape') diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..af20e54d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,23 @@ +# coding: utf-8 +import pytest +from requests.compat import urljoin + + +def prepare_url(value): + # Issue #1483: Make sure the URL always has a trailing slash + httpbin_url = value.url.rstrip('/') + '/' + + def inner(*suffix): + return urljoin(httpbin_url, '/'.join(suffix)) + + return inner + + +@pytest.fixture +def httpbin(httpbin): + return prepare_url(httpbin) + + +@pytest.fixture +def httpbin_secure(httpbin_secure): + return prepare_url(httpbin_secure) diff --git a/tests/test_hooks.py b/tests/test_hooks.py new file mode 100644 index 00000000..e2b174d8 --- /dev/null +++ b/tests/test_hooks.py @@ -0,0 +1,22 @@ +# coding: utf-8 +import pytest + +from requests import hooks + + +def hook(value): + return value[1:] + + +@pytest.mark.parametrize( + 'hooks_list, result', ( + (hook, 'ata'), + ([hook, lambda x: None, hook], 'ta'), + ) +) +def test_hooks(hooks_list, result): + assert hooks.dispatch_hook('response', {'response': hooks_list}, 'Data') == result + + +def test_default_hooks(): + assert hooks.default_hooks() == {'response': []} diff --git a/tests/test_lowlevel.py b/tests/test_lowlevel.py new file mode 100644 index 00000000..f3dd1b11 --- /dev/null +++ b/tests/test_lowlevel.py @@ -0,0 +1,56 @@ +import os +import pytest +import threading +import requests + +from tests.testserver.server import Server + +from .utils import override_environ + + +def test_chunked_upload(): + """can safely send generators""" + close_server = threading.Event() + server = Server.basic_response_server(wait_to_close_event=close_server) + data = iter([b'a', b'b', b'c']) + + with server as (host, port): + url = 'http://{0}:{1}/'.format(host, port) + r = requests.post(url, data=data, stream=True) + close_server.set() # release server block + + assert r.status_code == 200 + assert r.request.headers['Transfer-Encoding'] == 'chunked' + + +_schemes_by_var_prefix = [ + ('http', ['http']), + ('https', ['https']), + ('all', ['http', 'https']), +] + +_proxy_combos = [] +for prefix, schemes in _schemes_by_var_prefix: + for scheme in schemes: + _proxy_combos.append(("{0}_proxy".format(prefix), scheme)) + +_proxy_combos += [(var.upper(), scheme) for var, scheme in _proxy_combos] + + +@pytest.mark.parametrize("var,scheme", _proxy_combos) +def test_use_proxy_from_environment(httpbin, var, scheme): + url = "{0}://httpbin.org".format(scheme) + fake_proxy = Server() # do nothing with the requests; just close the socket + with fake_proxy as (host, port): + proxy_url = "socks5://{0}:{1}".format(host, port) + kwargs = {var: proxy_url} + with override_environ(**kwargs): + # fake proxy's lack of response will cause a ConnectionError + with pytest.raises(requests.exceptions.ConnectionError): + requests.get(url) + + # the fake proxy received a request + assert len(fake_proxy.handler_results) == 1 + + # it had actual content (not checking for SOCKS protocol for now) + assert len(fake_proxy.handler_results[0]) > 0 diff --git a/test_requests.py b/tests/test_requests.py similarity index 76% rename from test_requests.py rename to tests/test_requests.py index 8e97fbba..1e570223 100755 --- a/test_requests.py +++ b/tests/test_requests.py @@ -16,57 +16,21 @@ import pytest from requests.adapters import HTTPAdapter from requests.auth import HTTPDigestAuth, _basic_auth_str from requests.compat import ( - Morsel, cookielib, getproxies, str, urljoin, urlparse, is_py3, - builtin_str, OrderedDict, is_py2) + Morsel, cookielib, getproxies, str, urlparse, + builtin_str, OrderedDict) from requests.cookies import cookiejar_from_dict, morsel_to_cookie from requests.exceptions import ( - ConnectionError, ConnectTimeout, InvalidScheme, InvalidURL, MissingScheme, - ReadTimeout, Timeout, RetryError, TooManyRedirects) + ConnectionError, ConnectTimeout, InvalidSchema, InvalidURL, + MissingSchema, ReadTimeout, Timeout, RetryError, TooManyRedirects, + ProxyError) from requests.models import PreparedRequest from requests.structures import CaseInsensitiveDict from requests.sessions import SessionRedirectMixin from requests.models import urlencode from requests.hooks import default_hooks -try: - import StringIO -except ImportError: - import io as StringIO - -try: - from multiprocessing.pool import ThreadPool -except ImportError: - ThreadPool = None - -if is_py3: - def u(s): - return s -else: - def u(s): - return s.decode('unicode-escape') - - -@pytest.fixture -def httpbin(httpbin): - # Issue #1483: Make sure the URL always has a trailing slash - httpbin_url = httpbin.url.rstrip('/') + '/' - - def inner(*suffix): - return urljoin(httpbin_url, '/'.join(suffix)) - - return inner - - -@pytest.fixture -def httpsbin_url(httpbin_secure): - # Issue #1483: Make sure the URL always has a trailing slash - httpbin_url = httpbin_secure.url.rstrip('/') + '/' - - def inner(*suffix): - return urljoin(httpbin_url, '/'.join(suffix)) - - return inner - +from .compat import StringIO, u +from .utils import override_environ class SendRecordingAdapter(HTTPAdapter): """ @@ -85,7 +49,7 @@ class SendRecordingAdapter(HTTPAdapter): # Requests to this URL should always fail with a connection timeout (nothing # listening on that port) -TARPIT = "http://10.255.255.1" +TARPIT = 'http://10.255.255.1' class TestRequests: @@ -101,15 +65,14 @@ class TestRequests: requests.patch requests.post - @pytest.mark.parametrize('exception, url', - ( - (MissingScheme, 'hiwpefhipowhefopw'), - (InvalidScheme, 'localhost:3128'), - (InvalidScheme, 'localhost.localdomain:3128/'), - (InvalidScheme, '10.122.1.1:3128/'), + @pytest.mark.parametrize( + 'exception, url', ( + (MissingSchema, 'hiwpefhipowhefopw'), + (InvalidSchema, 'localhost:3128'), + (InvalidSchema, 'localhost.localdomain:3128/'), + (InvalidSchema, '10.122.1.1:3128/'), (InvalidURL, 'http://'), - ) - ) + )) def test_invalid_url(self, exception, url): with pytest.raises(exception): requests.get(url) @@ -141,12 +104,11 @@ class TestRequests: assert request.path_url == '/get/test%20case' - @pytest.mark.parametrize('url, expected', - ( + @pytest.mark.parametrize( + 'url, expected', ( ('http://example.com/path#fragment', 'http://example.com/path?a=b#fragment'), ('http://example.com/path?key=value#fragment', 'http://example.com/path?key=value&a=b#fragment') - ) - ) + )) def test_params_are_added_before_fragment(self, url, expected): request = requests.Request('GET', url, params={"a": "b"}).prepare() assert request.url == expected @@ -217,6 +179,49 @@ class TestRequests: else: pytest.fail('Expected custom max number of redirects to be respected but was not') + def test_http_301_changes_post_to_get(self, httpbin): + r = requests.post(httpbin('status', '301')) + assert r.status_code == 200 + assert r.request.method == 'GET' + assert r.history[0].status_code == 301 + assert r.history[0].is_redirect + + def test_http_301_doesnt_change_head_to_get(self, httpbin): + r = requests.head(httpbin('status', '301'), allow_redirects=True) + print(r.content) + assert r.status_code == 200 + assert r.request.method == 'HEAD' + assert r.history[0].status_code == 301 + assert r.history[0].is_redirect + + def test_http_302_changes_post_to_get(self, httpbin): + r = requests.post(httpbin('status', '302')) + assert r.status_code == 200 + assert r.request.method == 'GET' + assert r.history[0].status_code == 302 + assert r.history[0].is_redirect + + def test_http_302_doesnt_change_head_to_get(self, httpbin): + r = requests.head(httpbin('status', '302'), allow_redirects=True) + assert r.status_code == 200 + assert r.request.method == 'HEAD' + assert r.history[0].status_code == 302 + assert r.history[0].is_redirect + + def test_http_303_changes_post_to_get(self, httpbin): + r = requests.post(httpbin('status', '303')) + assert r.status_code == 200 + assert r.request.method == 'GET' + assert r.history[0].status_code == 303 + assert r.history[0].is_redirect + + def test_http_303_doesnt_change_head_to_get(self, httpbin): + r = requests.head(httpbin('status', '303'), allow_redirects=True) + assert r.status_code == 200 + assert r.request.method == 'HEAD' + assert r.history[0].status_code == 303 + assert r.history[0].is_redirect + # def test_HTTP_302_ALLOW_REDIRECT_POST(self): # r = requests.post(httpbin('status', '302'), data={'some': 'data'}) # self.assertEqual(r.status_code, 200) @@ -326,12 +331,30 @@ class TestRequests: prep = ses.prepare_request(req) assert 'Accept-Encoding' not in prep.headers + def test_headers_preserve_order(self, httpbin): + """Preserve order when headers provided as OrderedDict.""" + ses = requests.Session() + ses.headers = OrderedDict() + ses.headers['Accept-Encoding'] = 'identity' + ses.headers['First'] = '1' + ses.headers['Second'] = '2' + headers = OrderedDict([('Third', '3'), ('Fourth', '4')]) + headers['Fifth'] = '5' + headers['Second'] = '222' + req = requests.Request('GET', httpbin('get'), headers=headers) + prep = ses.prepare_request(req) + items = list(prep.headers.items()) + assert items[0] == ('Accept-Encoding', 'identity') + assert items[1] == ('First', '1') + assert items[2] == ('Second', '222') + assert items[3] == ('Third', '3') + assert items[4] == ('Fourth', '4') + assert items[5] == ('Fifth', '5') + @pytest.mark.parametrize('key', ('User-agent', 'user-agent')) def test_user_agent_transfers(self, httpbin, key): - heads = { - key: 'Mozilla/5.0 (github.com/kennethreitz/requests)' - } + heads = {key: 'Mozilla/5.0 (github.com/kennethreitz/requests)'} r = requests.get(httpbin('user-agent'), headers=heads) assert heads[key] in r.text @@ -359,20 +382,24 @@ class TestRequests: r = s.get(url) assert r.status_code == 200 - @pytest.mark.parametrize('url, exception', - ( + @pytest.mark.parametrize( + 'url, exception', ( # Connecting to an unknown domain should raise a ConnectionError ('http://doesnotexist.google.com', ConnectionError), # Connecting to an invalid port should raise a ConnectionError ('http://localhost:1', ConnectionError), # Inputing a URL that cannot be parsed should raise an InvalidURL error ('http://fe80::5054:ff:fe5a:fc0', InvalidURL) - ) - ) + )) def test_errors(self, url, exception): with pytest.raises(exception): requests.get(url, timeout=1) + def test_proxy_error(self): + # any proxy related error (address resolution, no route to host, etc) should result in a ProxyError + with pytest.raises(ProxyError): + requests.get('http://localhost:1', proxies={'http': 'non-resolvable-address'}) + def test_basicauth_with_netrc(self, httpbin): auth = ('user', 'pass') wrong_auth = ('wronguser', 'wrongpass') @@ -491,6 +518,48 @@ class TestRequests: with pytest.raises(ValueError): requests.post(url, files=['bad file data']) + def test_POSTBIN_SEEKED_OBJECT_WITH_NO_ITER(self, httpbin): + + class TestStream(object): + def __init__(self, data): + self.data = data.encode() + self.length = len(self.data) + self.index = 0 + + def __len__(self): + return self.length + + def read(self, size=None): + if size: + ret = self.data[self.index:self.index + size] + self.index += size + else: + ret = self.data[self.index:] + self.index = self.length + return ret + + def tell(self): + return self.index + + def seek(self, offset, where=0): + if where == 0: + self.index = offset + elif where == 1: + self.index += offset + elif where == 2: + self.index = self.length + offset + + test = TestStream('test') + post1 = requests.post(httpbin('post'), data=test) + assert post1.status_code == 200 + assert post1.json()['data'] == 'test' + + test = TestStream('test') + test.seek(2) + post2 = requests.post(httpbin('post'), data=test) + assert post2.status_code == 200 + assert post2.json()['data'] == 'st' + def test_POSTBIN_GET_POST_FILES_WITH_DATA(self, httpbin): url = httpbin('post') @@ -532,15 +601,14 @@ class TestRequests: r = requests.get(httpbin('gzip')) r.content.decode('ascii') - @pytest.mark.parametrize('url, params', - ( + @pytest.mark.parametrize( + 'url, params', ( ('/get', {'foo': 'føø'}), ('/get', {'føø': 'føø'}), ('/get', {'føø': 'føø'}), ('/get', {'foo': 'foo'}), ('ø', {'foo': 'foo'}), - ) - ) + )) def test_unicode_get(self, httpbin, url, params): requests.get(httpbin(url), params=params) @@ -550,8 +618,8 @@ class TestRequests: headers={str('Content-Type'): 'application/octet-stream'}, data='\xff') # compat.str is unicode. - def test_pyopenssl_redirect(self, httpsbin_url, httpbin_ca_bundle): - requests.get(httpsbin_url('status', '301'), verify=httpbin_ca_bundle) + def test_pyopenssl_redirect(self, httpbin_secure, httpbin_ca_bundle): + requests.get(httpbin_secure('status', '301'), verify=httpbin_ca_bundle) def test_urlencoded_get_query_multivalued_param(self, httpbin): @@ -572,8 +640,7 @@ class TestRequests: {'stuff': u('ëlïxr').encode('utf-8')}, {'stuff': 'elixr'}, {'stuff': 'elixr'.encode('utf-8')}, - ) - ) + )) def test_unicode_multipart_post(self, httpbin, data): r = requests.post(httpbin('post'), data=data, @@ -582,49 +649,59 @@ class TestRequests: def test_unicode_multipart_post_fieldnames(self, httpbin): filename = os.path.splitext(__file__)[0] + '.py' - r = requests.Request(method='POST', - url=httpbin('post'), - data={'stuff'.encode('utf-8'): 'elixr'}, - files={'file': ('test_requests.py', - open(filename, 'rb'))}) + r = requests.Request( + method='POST', url=httpbin('post'), + data={'stuff'.encode('utf-8'): 'elixr'}, + files={'file': ('test_requests.py', open(filename, 'rb'))}) prep = r.prepare() assert b'name="stuff"' in prep.body assert b'name="b\'stuff\'"' not in prep.body def test_unicode_method_name(self, httpbin): - files = {'file': open('test_requests.py', 'rb')} + files = {'file': open(__file__, 'rb')} r = requests.request( method=u('POST'), url=httpbin('post'), files=files) assert r.status_code == 200 def test_unicode_method_name_with_request_object(self, httpbin): - files = {'file': open('test_requests.py', 'rb')} + files = {'file': open(__file__, 'rb')} s = requests.Session() - req = requests.Request(u("POST"), httpbin('post'), files=files) + req = requests.Request(u('POST'), httpbin('post'), files=files) prep = s.prepare_request(req) assert isinstance(prep.method, builtin_str) - assert prep.method == "POST" + assert prep.method == 'POST' resp = s.send(prep) assert resp.status_code == 200 + def test_non_prepared_request_error(self): + s = requests.Session() + req = requests.Request(u('POST'), '/') + + with pytest.raises(ValueError) as e: + s.send(req) + assert str(e.value) == 'You can only send PreparedRequests.' + def test_custom_content_type(self, httpbin): r = requests.post( httpbin('post'), data={'stuff': json.dumps({'a': 123})}, - files={'file1': ('test_requests.py', open(__file__, 'rb')), - 'file2': ('test_requests', open(__file__, 'rb'), - 'text/py-content-type')}) + files={ + 'file1': ('test_requests.py', open(__file__, 'rb')), + 'file2': ('test_requests', open(__file__, 'rb'), + 'text/py-content-type')}) assert r.status_code == 200 assert b"text/py-content-type" in r.request.body def test_hook_receives_request_arguments(self, httpbin): def hook(resp, **kwargs): - # FIXME. Not executed assert resp is not None assert kwargs != {} - requests.Request('GET', httpbin(), hooks={'response': hook}) + s = requests.Session() + r = requests.Request('GET', httpbin(), hooks={'response': hook}) + prep = s.prepare_request(r) + s.send(prep) def test_session_hooks_are_used_with_no_request_hooks(self, httpbin): hook = lambda x, *args, **kwargs: x @@ -812,6 +889,38 @@ class TestRequests: # make sure one can use items multiple times assert list(items) == list(items) + def test_cookie_duplicate_names_different_domains(self): + key = 'some_cookie' + value = 'some_value' + domain1 = 'test1.com' + domain2 = 'test2.com' + + jar = requests.cookies.RequestsCookieJar() + jar.set(key, value, domain=domain1) + jar.set(key, value, domain=domain2) + assert key in jar + items = jar.items() + assert len(items) == 2 + + # Verify that CookieConflictError is raised if domain is not specified + with pytest.raises(requests.cookies.CookieConflictError): + jar.get(key) + + # Verify that CookieConflictError is not raised if domain is specified + cookie = jar.get(key, domain=domain1) + assert cookie == value + + def test_cookie_duplicate_names_raises_cookie_conflict_error(self): + key = 'some_cookie' + value = 'some_value' + path = 'some_path' + + jar = requests.cookies.RequestsCookieJar() + jar.set(key, value, path=path) + jar.set(key, value) + with pytest.raises(requests.cookies.CookieConflictError): + jar.get(key) + def test_time_elapsed_blank(self, httpbin): r = requests.get(httpbin('get')) td = r.elapsed @@ -863,26 +972,6 @@ class TestRequests: assert r.request.url == pr.request.url assert r.request.headers == pr.request.headers - def test_get_auth_from_url(self): - url = 'http://user:pass@complex.url.com/path?query=yes' - assert ('user', 'pass') == requests.utils.get_auth_from_url(url) - - def test_get_auth_from_url_encoded_spaces(self): - url = 'http://user:pass%20pass@complex.url.com/path?query=yes' - assert ('user', 'pass pass') == requests.utils.get_auth_from_url(url) - - def test_get_auth_from_url_not_encoded_spaces(self): - url = 'http://user:pass pass@complex.url.com/path?query=yes' - assert ('user', 'pass pass') == requests.utils.get_auth_from_url(url) - - def test_get_auth_from_url_percent_chars(self): - url = 'http://user%25user:pass@complex.url.com/path?query=yes' - assert ('user%user', 'pass') == requests.utils.get_auth_from_url(url) - - def test_get_auth_from_url_encoded_hashes(self): - url = 'http://user:pass%23pass@complex.url.com/path?query=yes' - assert ('user', 'pass#pass') == requests.utils.get_auth_from_url(url) - def test_cannot_send_unprepared_requests(self, httpbin): r = requests.Request(url=httpbin()) with pytest.raises(ValueError): @@ -998,23 +1087,10 @@ class TestRequests: assert 'unicode' in p.headers.keys() assert 'byte' in p.headers.keys() - def test_can_send_nonstring_objects_with_files(self, httpbin): - data = {'a': 0.0} - files = {'b': 'foo'} - r = requests.Request('POST', httpbin('post'), data=data, files=files) - p = r.prepare() - - assert 'multipart/form-data' in p.headers['Content-Type'] - - def test_can_send_bytes_bytearray_objects_with_files(self, httpbin): - # Test bytes: + @pytest.mark.parametrize('files', ('foo', b'foo', bytearray(b'foo'))) + def test_can_send_objects_with_files(self, httpbin, files): data = {'a': 'this is a string'} - files = {'b': b'foo'} - r = requests.Request('POST', httpbin('post'), data=data, files=files) - p = r.prepare() - assert 'multipart/form-data' in p.headers['Content-Type'] - # Test bytearrays: - files = {'b': bytearray(b'foo')} + files = {'b': files} r = requests.Request('POST', httpbin('post'), data=data, files=files) p = r.prepare() assert 'multipart/form-data' in p.headers['Content-Type'] @@ -1194,48 +1270,37 @@ class TestRequests: with pytest.raises(KeyError): proxies['http'] -class TestContentEncodingDetection: + def test_session_close_proxy_clear(self, mocker): + proxies = { + 'one': mocker.Mock(), + 'two': mocker.Mock(), + } + session = requests.Session() + mocker.patch.dict(session.adapters['http://'].proxy_manager, proxies) + session.close() + proxies['one'].clear.assert_called_once_with() + proxies['two'].clear.assert_called_once_with() - def test_none(self): - encodings = requests.utils.get_encodings_from_content('') - assert not len(encodings) + def test_response_json_when_content_is_None(self, httpbin): + r = requests.get(httpbin('/status/204')) + # Make sure r.content is None + r.status_code = 0 + r._content = False + r._content_consumed = False - @pytest.mark.parametrize('content', - ( - # HTML5 meta charset attribute - '', - # HTML4 pragma directive - '', - # XHTML 1.x served with text/html MIME type - '', - # XHTML 1.x served as XML - '', - ) - ) - def test_pragmas(self, content): - encodings = requests.utils.get_encodings_from_content(content) - assert len(encodings) == 1 - assert encodings[0] == 'UTF-8' - - def test_precedence(self): - content = ''' - - - - '''.strip() - encodings = requests.utils.get_encodings_from_content(content) - assert encodings == ['HTML5', 'HTML4', 'XML'] + assert r.content is None + with pytest.raises(ValueError): + r.json() class TestCaseInsensitiveDict: - @pytest.mark.parametrize('cid', - ( + @pytest.mark.parametrize( + 'cid', ( CaseInsensitiveDict({'Foo': 'foo', 'BAr': 'bar'}), CaseInsensitiveDict([('Foo', 'foo'), ('BAr', 'bar')]), CaseInsensitiveDict(FOO='foo', BAr='bar'), - ) - ) + )) def test_init(self, cid): assert len(cid) == 2 assert 'foo' in cid @@ -1381,159 +1446,6 @@ class TestCaseInsensitiveDict: # The whitespaces can't be in the middle of the URL though: assert requests.get(get_url + ' abc').status_code == 404 -class TestUtils: - - def test_super_len_io_streams(self): - """ Ensures that we properly deal with different kinds of IO streams. """ - # uses StringIO or io.StringIO (see import above) - from io import BytesIO - from requests.utils import super_len - - assert super_len(StringIO.StringIO()) == 0 - assert super_len( - StringIO.StringIO('with so much drama in the LBC')) == 29 - - assert super_len(BytesIO()) == 0 - assert super_len( - BytesIO(b"it's kinda hard bein' snoop d-o-double-g")) == 40 - - try: - import cStringIO - except ImportError: - pass - else: - assert super_len( - cStringIO.StringIO('but some how, some way...')) == 25 - - def test_super_len_correctly_calculates_len_of_partially_read_file(self): - """Ensure that we handle partially consumed file like objects.""" - from requests.utils import super_len - s = StringIO.StringIO() - s.write('foobarbogus') - assert super_len(s) == 0 - - def test_get_environ_proxies_ip_ranges(self): - """Ensures that IP addresses are correctly matches with ranges - in no_proxy variable.""" - from requests.utils import get_environ_proxies - os.environ['no_proxy'] = "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1" - assert get_environ_proxies('http://192.168.0.1:5000/') == {} - assert get_environ_proxies('http://192.168.0.1/') == {} - assert get_environ_proxies('http://172.16.1.1/') == {} - assert get_environ_proxies('http://172.16.1.1:5000/') == {} - assert get_environ_proxies('http://192.168.1.1:5000/') != {} - assert get_environ_proxies('http://192.168.1.1/') != {} - - def test_get_environ_proxies(self): - """Ensures that IP addresses are correctly matches with ranges - in no_proxy variable.""" - from requests.utils import get_environ_proxies - os.environ['no_proxy'] = "127.0.0.1,localhost.localdomain,192.168.0.0/24,172.16.1.1" - assert get_environ_proxies( - 'http://localhost.localdomain:5000/v1.0/') == {} - assert get_environ_proxies('http://www.requests.com/') != {} - - def test_select_proxies(self): - """Make sure we can select per-host proxies correctly.""" - from requests.utils import select_proxy - proxies = {'http': 'http://http.proxy', - 'http://some.host': 'http://some.host.proxy'} - assert select_proxy('hTTp://u:p@Some.Host/path', proxies) == 'http://some.host.proxy' - assert select_proxy('hTTp://u:p@Other.Host/path', proxies) == 'http://http.proxy' - assert select_proxy('hTTps://Other.Host', proxies) is None - - def test_guess_filename_when_int(self): - from requests.utils import guess_filename - assert None is guess_filename(1) - - def test_guess_filename_when_filename_is_an_int(self): - from requests.utils import guess_filename - fake = type('Fake', (object,), {'name': 1})() - assert None is guess_filename(fake) - - def test_guess_filename_with_file_like_obj(self): - from requests.utils import guess_filename - from requests import compat - fake = type('Fake', (object,), {'name': b'value'})() - guessed_name = guess_filename(fake) - assert b'value' == guessed_name - assert isinstance(guessed_name, compat.bytes) - - def test_guess_filename_with_unicode_name(self): - from requests.utils import guess_filename - from requests import compat - filename = b'value'.decode('utf-8') - fake = type('Fake', (object,), {'name': filename})() - guessed_name = guess_filename(fake) - assert filename == guessed_name - assert isinstance(guessed_name, compat.str) - - def test_is_ipv4_address(self): - from requests.utils import is_ipv4_address - assert is_ipv4_address('8.8.8.8') - assert not is_ipv4_address('8.8.8.8.8') - assert not is_ipv4_address('localhost.localdomain') - - def test_is_valid_cidr(self): - from requests.utils import is_valid_cidr - assert not is_valid_cidr('8.8.8.8') - assert is_valid_cidr('192.168.1.0/24') - - def test_dotted_netmask(self): - from requests.utils import dotted_netmask - assert dotted_netmask(8) == '255.0.0.0' - assert dotted_netmask(24) == '255.255.255.0' - assert dotted_netmask(25) == '255.255.255.128' - - def test_address_in_network(self): - from requests.utils import address_in_network - assert address_in_network('192.168.1.1', '192.168.1.0/24') - assert not address_in_network('172.16.0.1', '192.168.1.0/24') - - def test_get_auth_from_url(self): - """Ensures that username and password in well-encoded URI as per - RFC 3986 are correctly extracted.""" - from requests.utils import get_auth_from_url - from requests.compat import quote - percent_encoding_test_chars = "%!*'();:@&=+$,/?#[] " - url_address = "request.com/url.html#test" - url = "http://" + quote( - percent_encoding_test_chars, '') + ':' + quote( - percent_encoding_test_chars, '') + '@' + url_address - (username, password) = get_auth_from_url(url) - assert username == percent_encoding_test_chars - assert password == percent_encoding_test_chars - - def test_requote_uri_with_unquoted_percents(self): - """Ensure we handle unquoted percent signs in redirects. - - See: https://github.com/kennethreitz/requests/issues/2356 - """ - from requests.utils import requote_uri - bad_uri = 'http://example.com/fiz?buz=%ppicture' - quoted = 'http://example.com/fiz?buz=%25ppicture' - assert quoted == requote_uri(bad_uri) - - def test_requote_uri_properly_requotes(self): - """Ensure requoting doesn't break expectations.""" - from requests.utils import requote_uri - quoted = 'http://example.com/fiz?buz=%25ppicture' - assert quoted == requote_uri(quoted) - - def test_unquote_unreserved_handles_unicode(self): - """Unicode strings can be passed to unquote_unreserved""" - from requests.utils import unquote_unreserved - uri = u'http://example.com/fizz?buzz=%41%2C' - unquoted = u'http://example.com/fizz?buzz=A%2C' - assert unquoted == unquote_unreserved(uri) - - def test_unquote_unreserved_handles_bytes(self): - """Bytestrings can be passed to unquote_unreserved""" - from requests.utils import unquote_unreserved - uri = b'http://example.com/fizz?buzz=%41%2C' - unquoted = b'http://example.com/fizz?buzz=A%2C' - assert unquoted == unquote_unreserved(uri) - class TestMorselToCookieExpires: """Tests for morsel_to_cookie when morsel contains expires.""" @@ -1546,12 +1458,11 @@ class TestMorselToCookieExpires: cookie = morsel_to_cookie(morsel) assert cookie.expires == 1 - @pytest.mark.parametrize('value, exception', - ( + @pytest.mark.parametrize( + 'value, exception', ( (100, TypeError), ('woops', ValueError), - ) - ) + )) def test_expires_invalid_int(self, value, exception): """Test case where an invalid type is passed for expires.""" morsel = Morsel() @@ -1590,20 +1501,22 @@ class TestMorselToCookieMaxAge: class TestTimeout: + def test_stream_timeout(self, httpbin): try: requests.get(httpbin('delay/10'), timeout=2.0) except requests.exceptions.Timeout as e: assert 'Read timed out' in e.args[0].args[0] - def test_invalid_timeout(self, httpbin): + @pytest.mark.parametrize( + 'timeout, error_text', ( + ((3, 4, 5), '(connect, read)'), + ('foo', 'must be an int or float'), + )) + def test_invalid_timeout(self, httpbin, timeout, error_text): with pytest.raises(ValueError) as e: - requests.get(httpbin('get'), timeout=(3, 4, 5)) - assert '(connect, read)' in str(e) - - with pytest.raises(ValueError) as e: - requests.get(httpbin('get'), timeout="foo") - assert 'must be an int or float' in str(e) + requests.get(httpbin('get'), timeout=timeout) + assert error_text in str(e) def test_none_timeout(self, httpbin): """ Check that you can set None as a valid timeout value. @@ -1681,7 +1594,25 @@ class RedirectSession(SessionRedirectMixin): return string -class TestRedirects: +def test_json_encodes_as_bytes(): + # urllib3 expects bodies as bytes-like objects + body = {"key": "value"} + p = PreparedRequest() + p.prepare( + method='GET', + url='https://www.example.com/', + json=body + ) + assert isinstance(p.body, bytes) + + +def test_requests_are_updated_each_time(httpbin): + session = RedirectSession([303, 307]) + prep = requests.Request('POST', httpbin('post')).prepare() + r0 = session.send(prep) + assert r0.request.method == 'POST' + assert session.calls[-1] == SendCall((r0.request,), {}) + redirect_generator = session.resolve_redirects(r0, prep) default_keyword_args = { 'stream': False, 'verify': True, @@ -1690,97 +1621,81 @@ class TestRedirects: 'allow_redirects': False, 'proxies': {}, } + for response in redirect_generator: + assert response.request.method == 'GET' + send_call = SendCall((response.request,), default_keyword_args) + assert session.calls[-1] == send_call - def test_requests_are_updated_each_time(self, httpbin): - session = RedirectSession([303, 307]) - prep = requests.Request('POST', httpbin('post')).prepare() - r0 = session.send(prep) - assert r0.request.method == 'POST' - assert session.calls[-1] == SendCall((r0.request,), {}) - redirect_generator = session.resolve_redirects(r0) - for response in redirect_generator: - assert response.request.method == 'GET' - send_call = SendCall((response.request,), - TestRedirects.default_keyword_args) - assert session.calls[-1] == send_call - @pytest.mark.skipif(is_py2, reason="requires python 3") - def test_redirects_with_latin1_header(self, httpbin): - """Test that redirect headers decoded with Latin 1 are correctly - followed""" - session = RedirectSession([303]) - session.location = u'http://xn--n8jyd3c767qtje.xn--q9jyb4c/ã\x83\x96ã\x83\xadã\x82°/' - prep = requests.Request('GET', httpbin('get')).prepare() - r0 = session.send(prep) +@pytest.mark.parametrize("var,url,proxy", [ + ('http_proxy', 'http://example.com', 'socks5://proxy.com:9876'), + ('https_proxy', 'https://example.com', 'socks5://proxy.com:9876'), + ('all_proxy', 'http://example.com', 'socks5://proxy.com:9876'), + ('all_proxy', 'https://example.com', 'socks5://proxy.com:9876'), +]) +def test_proxy_env_vars_override_default(var, url, proxy): + session = requests.Session() + prep = PreparedRequest() + prep.prepare(method='GET', url=url) - responses = list(session.resolve_redirects(r0, prep)) - assert len(responses) == 1 - assert responses[0].request.url == u'http://xn--n8jyd3c767qtje.xn--q9jyb4c/%E3%83%96%E3%83%AD%E3%82%B0/' + kwargs = { + var: proxy + } + scheme = urlparse(url).scheme + with override_environ(**kwargs): + proxies = session.rebuild_proxies(prep, {}) + assert scheme in proxies + assert proxies[scheme] == proxy -@pytest.fixture -def list_of_tuples(): - return [ + +@pytest.mark.parametrize( + 'data', ( (('a', 'b'), ('c', 'd')), (('c', 'd'), ('a', 'b')), (('a', 'b'), ('c', 'd'), ('e', 'f')), - ] - - -def test_data_argument_accepts_tuples(list_of_tuples): + )) +def test_data_argument_accepts_tuples(data): """Ensure that the data argument will accept tuples of strings and properly encode them. """ - for data in list_of_tuples: - p = PreparedRequest() - p.prepare( - method='GET', - url='http://www.example.com', - data=data, - hooks=default_hooks() - ) - assert p.body == urlencode(data) - - -def assert_copy(p, p_copy): - for attr in ('method', 'url', 'headers', '_cookies', 'body', 'hooks'): - assert getattr(p, attr) == getattr(p_copy, attr) - - -def test_prepared_request_empty_copy(): - p = PreparedRequest() - assert_copy(p, p.copy()) - - -def test_prepared_request_no_cookies_copy(): p = PreparedRequest() p.prepare( method='GET', url='http://www.example.com', - data='foo=bar', + data=data, hooks=default_hooks() ) - assert_copy(p, p.copy()) + assert p.body == urlencode(data) -def test_prepared_request_complete_copy(): +@pytest.mark.parametrize( + 'kwargs', ( + None, + { + 'method': 'GET', + 'url': 'http://www.example.com', + 'data': 'foo=bar', + 'hooks': default_hooks() + }, + { + 'method': 'GET', + 'url': 'http://www.example.com', + 'data': 'foo=bar', + 'hooks': default_hooks(), + 'cookies': {'foo': 'bar'} + }, + { + 'method': 'GET', + 'url': u('http://www.example.com/üniçø∂é') + }, + )) +def test_prepared_copy(kwargs): p = PreparedRequest() - p.prepare( - method='GET', - url='http://www.example.com', - data='foo=bar', - hooks=default_hooks(), - cookies={'foo': 'bar'} - ) - assert_copy(p, p.copy()) - - -def test_prepare_unicode_url(): - p = PreparedRequest() - p.prepare( - method='GET', - url=u('http://www.example.com/üniçø∂é'), - ) - assert_copy(p, p.copy()) + if kwargs: + p.prepare(**kwargs) + copy = p.copy() + for attr in ('method', 'url', 'headers', '_cookies', 'body', 'hooks'): + assert getattr(p, attr) == getattr(copy, attr) def test_prepare_requires_a_request_method(): diff --git a/tests/test_structures.py b/tests/test_structures.py new file mode 100644 index 00000000..1c332bb2 --- /dev/null +++ b/tests/test_structures.py @@ -0,0 +1,79 @@ +# coding: utf-8 +import pytest + +from requests.structures import CaseInsensitiveDict, LookupDict + + +class TestCaseInsensitiveDict: + + @pytest.fixture(autouse=True) + def setup(self): + """ + CaseInsensitiveDict instance with "Accept" header. + """ + self.case_insensitive_dict = CaseInsensitiveDict() + self.case_insensitive_dict['Accept'] = 'application/json' + + def test_list(self): + assert list(self.case_insensitive_dict) == ['Accept'] + + possible_keys = pytest.mark.parametrize('key', ('accept', 'ACCEPT', 'aCcEpT', 'Accept')) + + @possible_keys + def test_getitem(self, key): + assert self.case_insensitive_dict[key] == 'application/json' + + @possible_keys + def test_delitem(self, key): + del self.case_insensitive_dict[key] + assert key not in self.case_insensitive_dict + + def test_lower_items(self): + assert list(self.case_insensitive_dict.lower_items()) == [('accept', 'application/json')] + + def test_repr(self): + assert repr(self.case_insensitive_dict) == "{'Accept': 'application/json'}" + + def test_copy(self): + copy = self.case_insensitive_dict.copy() + assert copy is not self.case_insensitive_dict + assert copy == self.case_insensitive_dict + + @pytest.mark.parametrize( + 'other, result', ( + ({'AccePT': 'application/json'}, True), + ({}, False), + (None, False) + ) + ) + def test_instance_equality(self, other, result): + assert (self.case_insensitive_dict == other) is result + + +class TestLookupDict: + + @pytest.fixture(autouse=True) + def setup(self): + """ + LookupDict instance with "bad_gateway" attribute. + """ + self.lookup_dict = LookupDict('test') + self.lookup_dict.bad_gateway = 502 + + def test_repr(self): + assert repr(self.lookup_dict) == "" + + get_item_parameters = pytest.mark.parametrize( + 'key, value', ( + ('bad_gateway', 502), + ('not_a_key', None) + ) + ) + + @get_item_parameters + def test_getitem(self, key, value): + assert self.lookup_dict[key] == value + + @get_item_parameters + def test_get(self, key, value): + assert self.lookup_dict.get(key) == value diff --git a/tests/test_testserver.py b/tests/test_testserver.py new file mode 100644 index 00000000..9a35460e --- /dev/null +++ b/tests/test_testserver.py @@ -0,0 +1,160 @@ +import threading +import socket +import time + +import pytest +import requests +from tests.testserver.server import Server + +class TestTestServer: + def test_basic(self): + """messages are sent and received properly""" + question = b"sucess?" + answer = b"yeah, success" + def handler(sock): + text = sock.recv(1000) + assert text == question + sock.sendall(answer) + + with Server(handler) as (host, port): + sock = socket.socket() + sock.connect((host, port)) + sock.sendall(question) + text = sock.recv(1000) + assert text == answer + sock.close() + + def test_server_closes(self): + """the server closes when leaving the context manager""" + with Server.basic_response_server() as (host, port): + sock = socket.socket() + sock.connect((host, port)) + + sock.close() + + with pytest.raises(socket.error): + new_sock = socket.socket() + new_sock.connect((host, port)) + + def test_text_response(self): + """the text_response_server sends the given text""" + server = Server.text_response_server( + "HTTP/1.1 200 OK\r\n" + + "Content-Length: 6\r\n" + + "\r\nroflol" + ) + + with server as (host, port): + r = requests.get('http://{0}:{1}'.format(host, port)) + + assert r.status_code == 200 + assert r.text == u'roflol' + assert r.headers['Content-Length'] == '6' + + def test_basic_response(self): + """the basic response server returns an empty http response""" + with Server.basic_response_server() as (host, port): + r = requests.get('http://{0}:{1}'.format(host, port)) + assert r.status_code == 200 + assert r.text == u'' + assert r.headers['Content-Length'] == '0' + + def test_basic_waiting_server(self): + """the server waits for the block_server event to be set before closing""" + block_server = threading.Event() + + with Server.basic_response_server(wait_to_close_event=block_server) as (host, port): + sock = socket.socket() + sock.connect((host, port)) + sock.sendall(b'send something') + time.sleep(2.5) + sock.sendall(b'still alive') + block_server.set() # release server block + + def test_multiple_requests(self): + """multiple requests can be served""" + requests_to_handle = 5 + + server = Server.basic_response_server(requests_to_handle=requests_to_handle) + + with server as (host, port): + server_url = 'http://{0}:{1}'.format(host, port) + for _ in range(requests_to_handle): + r = requests.get(server_url) + assert r.status_code == 200 + + # the (n+1)th request fails + with pytest.raises(requests.exceptions.ConnectionError): + r = requests.get(server_url) + + def test_request_recovery(self): + """can check the requests content""" + server = Server.basic_response_server(requests_to_handle=2) + first_request = b'put your hands up in the air' + second_request = b'put your hand down in the floor' + + with server as address: + sock1 = socket.socket() + sock2 = socket.socket() + + sock1.connect(address) + sock1.sendall(first_request) + sock1.close() + + sock2.connect(address) + sock2.sendall(second_request) + sock2.close() + + assert server.handler_results[0] == first_request + assert server.handler_results[1] == second_request + + def test_requests_after_timeout_are_not_received(self): + """the basic response handler times out when receiving requests""" + server = Server.basic_response_server(request_timeout=1) + + with server as address: + sock = socket.socket() + sock.connect(address) + time.sleep(1.5) + sock.sendall(b'hehehe, not received') + sock.close() + + assert server.handler_results[0] == b'' + + + def test_request_recovery_with_bigger_timeout(self): + """a biggest timeout can be specified""" + server = Server.basic_response_server(request_timeout=3) + data = b'bananadine' + + with server as address: + sock = socket.socket() + sock.connect(address) + time.sleep(1.5) + sock.sendall(data) + sock.close() + + assert server.handler_results[0] == data + + def test_server_finishes_on_error(self): + """the server thread exits even if an exception exits the context manager""" + server = Server.basic_response_server() + with pytest.raises(Exception): + with server: + raise Exception() + + assert len(server.handler_results) == 0 + + # if the server thread fails to finish, the test suite will hang + # and get killed by the jenkins timeout. + + def test_server_finishes_when_no_connections(self): + """the server thread exits even if there are no connections""" + server = Server.basic_response_server() + with server: + pass + + assert len(server.handler_results) == 0 + + # if the server thread fails to finish, the test suite will hang + # and get killed by the jenkins timeout. diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..17149d26 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,461 @@ +# coding: utf-8 +from io import BytesIO + +import pytest +from requests import compat +from requests.structures import CaseInsensitiveDict +from requests.utils import ( + address_in_network, dotted_netmask, + get_auth_from_url, get_encoding_from_headers, + get_encodings_from_content, get_environ_proxies, + guess_filename, guess_json_utf, is_ipv4_address, + is_valid_cidr, iter_slices, parse_dict_header, + parse_header_links, prepend_scheme_if_needed, + requote_uri, select_proxy, should_bypass_proxies, super_len, + to_key_val_list, to_native_string, + unquote_header_value, unquote_unreserved, + urldefragauth) + +from .compat import StringIO, cStringIO + + +class TestSuperLen: + + @pytest.mark.parametrize( + 'stream, value', ( + (StringIO.StringIO, 'Test'), + (BytesIO, b'Test'), + pytest.mark.skipif('cStringIO is None')((cStringIO, 'Test')), + )) + def test_io_streams(self, stream, value): + """Ensures that we properly deal with different kinds of IO streams.""" + assert super_len(stream()) == 0 + assert super_len(stream(value)) == 4 + + def test_super_len_correctly_calculates_len_of_partially_read_file(self): + """Ensure that we handle partially consumed file like objects.""" + s = StringIO.StringIO() + s.write('foobarbogus') + assert super_len(s) == 0 + + @pytest.mark.parametrize('error', [IOError, OSError]) + def test_super_len_handles_files_raising_weird_errors_in_tell(self, error): + """ + If tell() raises errors, assume the cursor is at position zero. + """ + class BoomFile(object): + def __len__(self): + return 5 + + def tell(self): + raise error() + + assert super_len(BoomFile()) == 0 + + def test_string(self): + assert super_len('Test') == 4 + + @pytest.mark.parametrize( + 'mode, warnings_num', ( + ('r', 1), + ('rb', 0), + )) + def test_file(self, tmpdir, mode, warnings_num, recwarn): + file_obj = tmpdir.join('test.txt') + file_obj.write('Test') + with file_obj.open(mode) as fd: + assert super_len(fd) == 4 + assert len(recwarn) == warnings_num + + +class TestToKeyValList: + + @pytest.mark.parametrize( + 'value, expected', ( + ([('key', 'val')], [('key', 'val')]), + ((('key', 'val'), ), [('key', 'val')]), + ({'key': 'val'}, [('key', 'val')]), + (None, None) + )) + def test_valid(self, value, expected): + assert to_key_val_list(value) == expected + + def test_invalid(self): + with pytest.raises(ValueError): + to_key_val_list('string') + + +class TestUnquoteHeaderValue: + + @pytest.mark.parametrize( + 'value, expected', ( + (None, None), + ('Test', 'Test'), + ('"Test"', 'Test'), + ('"Test\\\\"', 'Test\\'), + ('"\\\\Comp\\Res"', '\\Comp\\Res'), + )) + def test_valid(self, value, expected): + assert unquote_header_value(value) == expected + + def test_is_filename(self): + assert unquote_header_value('"\\\\Comp\\Res"', True) == '\\\\Comp\\Res' + + +class TestGetEnvironProxies: + """Ensures that IP addresses are correctly matches with ranges + in no_proxy variable.""" + + @pytest.fixture(autouse=True, params=['no_proxy', 'NO_PROXY']) + def no_proxy(self, request, monkeypatch): + monkeypatch.setenv(request.param, '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') + + @pytest.mark.parametrize( + 'url', ( + 'http://192.168.0.1:5000/', + 'http://192.168.0.1/', + 'http://172.16.1.1/', + 'http://172.16.1.1:5000/', + 'http://localhost.localdomain:5000/v1.0/', + )) + def test_bypass(self, url): + assert get_environ_proxies(url) == {} + + @pytest.mark.parametrize( + 'url', ( + 'http://192.168.1.1:5000/', + 'http://192.168.1.1/', + 'http://www.requests.com/', + )) + def test_not_bypass(self, url): + assert get_environ_proxies(url) != {} + + +class TestIsIPv4Address: + + def test_valid(self): + assert is_ipv4_address('8.8.8.8') + + @pytest.mark.parametrize('value', ('8.8.8.8.8', 'localhost.localdomain')) + def test_invalid(self, value): + assert not is_ipv4_address(value) + + +class TestIsValidCIDR: + + def test_valid(self): + assert is_valid_cidr('192.168.1.0/24') + + @pytest.mark.parametrize( + 'value', ( + '8.8.8.8', + '192.168.1.0/a', + '192.168.1.0/128', + '192.168.1.0/-1', + '192.168.1.999/24', + )) + def test_invalid(self, value): + assert not is_valid_cidr(value) + + +class TestAddressInNetwork: + + def test_valid(self): + assert address_in_network('192.168.1.1', '192.168.1.0/24') + + def test_invalid(self): + assert not address_in_network('172.16.0.1', '192.168.1.0/24') + + +class TestGuessFilename: + + @pytest.mark.parametrize( + 'value', (1, type('Fake', (object,), {'name': 1})()), + ) + def test_guess_filename_invalid(self, value): + assert guess_filename(value) is None + + @pytest.mark.parametrize( + 'value, expected_type', ( + (b'value', compat.bytes), + (b'value'.decode('utf-8'), compat.str) + )) + def test_guess_filename_valid(self, value, expected_type): + obj = type('Fake', (object,), {'name': value})() + result = guess_filename(obj) + assert result == value + assert isinstance(result, expected_type) + + +class TestContentEncodingDetection: + + def test_none(self): + encodings = get_encodings_from_content('') + assert not len(encodings) + + @pytest.mark.parametrize( + 'content', ( + # HTML5 meta charset attribute + '', + # HTML4 pragma directive + '', + # XHTML 1.x served with text/html MIME type + '', + # XHTML 1.x served as XML + '', + )) + def test_pragmas(self, content): + encodings = get_encodings_from_content(content) + assert len(encodings) == 1 + assert encodings[0] == 'UTF-8' + + def test_precedence(self): + content = ''' + + + + '''.strip() + assert get_encodings_from_content(content) == ['HTML5', 'HTML4', 'XML'] + + +class TestGuessJSONUTF: + + @pytest.mark.parametrize( + 'encoding', ( + 'utf-32', 'utf-8-sig', 'utf-16', 'utf-8', 'utf-16-be', 'utf-16-le', + 'utf-32-be', 'utf-32-le' + )) + def test_encoded(self, encoding): + data = '{}'.encode(encoding) + assert guess_json_utf(data) == encoding + + def test_bad_utf_like_encoding(self): + assert guess_json_utf(b'\x00\x00\x00\x00') is None + + +USER = PASSWORD = "%!*'();:@&=+$,/?#[] " +ENCODED_USER = compat.quote(USER, '') +ENCODED_PASSWORD = compat.quote(PASSWORD, '') + + +@pytest.mark.parametrize( + 'url, auth', ( + ( + 'http://' + ENCODED_USER + ':' + ENCODED_PASSWORD + '@' + + 'request.com/url.html#test', + (USER, PASSWORD) + ), + ( + 'http://user:pass@complex.url.com/path?query=yes', + ('user', 'pass') + ), + ( + 'http://user:pass%20pass@complex.url.com/path?query=yes', + ('user', 'pass pass') + ), + ( + 'http://user:pass pass@complex.url.com/path?query=yes', + ('user', 'pass pass') + ), + ( + 'http://user%25user:pass@complex.url.com/path?query=yes', + ('user%user', 'pass') + ), + ( + 'http://user:pass%23pass@complex.url.com/path?query=yes', + ('user', 'pass#pass') + ), + ( + 'http://complex.url.com/path?query=yes', + ('', '') + ), + )) +def test_get_auth_from_url(url, auth): + assert get_auth_from_url(url) == auth + + +@pytest.mark.parametrize( + 'uri, expected', ( + ( + # Ensure requoting doesn't break expectations + 'http://example.com/fiz?buz=%25ppicture', + 'http://example.com/fiz?buz=%25ppicture', + ), + ( + # Ensure we handle unquoted percent signs in redirects + 'http://example.com/fiz?buz=%ppicture', + 'http://example.com/fiz?buz=%25ppicture', + ), + )) +def test_requote_uri_with_unquoted_percents(uri, expected): + """See: https://github.com/kennethreitz/requests/issues/2356 + """ + assert requote_uri(uri) == expected + + +@pytest.mark.parametrize( + 'uri, expected', ( + ( + # Illegal bytes + 'http://example.com/?a=%--', + 'http://example.com/?a=%--', + ), + ( + # Reserved characters + 'http://example.com/?a=%300', + 'http://example.com/?a=00', + ) + )) +def test_unquote_unreserved(uri, expected): + assert unquote_unreserved(uri) == expected + + +@pytest.mark.parametrize( + 'mask, expected', ( + (8, '255.0.0.0'), + (24, '255.255.255.0'), + (25, '255.255.255.128'), + )) +def test_dotted_netmask(mask, expected): + assert dotted_netmask(mask) == expected + + +http_proxies = {'http': 'http://http.proxy', + 'http://some.host': 'http://some.host.proxy'} +all_proxies = {'all': 'socks5://http.proxy', + 'all://some.host': 'socks5://some.host.proxy'} +@pytest.mark.parametrize( + 'url, expected, proxies', ( + ('hTTp://u:p@Some.Host/path', 'http://some.host.proxy', http_proxies), + ('hTTp://u:p@Other.Host/path', 'http://http.proxy', http_proxies), + ('hTTp:///path', 'http://http.proxy', http_proxies), + ('hTTps://Other.Host', None, http_proxies), + ('file:///etc/motd', None, http_proxies), + + ('hTTp://u:p@Some.Host/path', 'socks5://some.host.proxy', all_proxies), + ('hTTp://u:p@Other.Host/path', 'socks5://http.proxy', all_proxies), + ('hTTp:///path', 'socks5://http.proxy', all_proxies), + ('hTTps://Other.Host', 'socks5://http.proxy', all_proxies), + + # XXX: unsure whether this is reasonable behavior + ('file:///etc/motd', 'socks5://http.proxy', all_proxies), + )) +def test_select_proxies(url, expected, proxies): + """Make sure we can select per-host proxies correctly.""" + assert select_proxy(url, proxies) == expected + + +@pytest.mark.parametrize( + 'value, expected', ( + ('foo="is a fish", bar="as well"', {'foo': 'is a fish', 'bar': 'as well'}), + ('key_without_value', {'key_without_value': None}) + )) +def test_parse_dict_header(value, expected): + assert parse_dict_header(value) == expected + + +@pytest.mark.parametrize( + 'value, expected', ( + ( + CaseInsensitiveDict(), + None + ), + ( + CaseInsensitiveDict({'content-type': 'application/json; charset=utf-8'}), + 'utf-8' + ), + ( + CaseInsensitiveDict({'content-type': 'text/plain'}), + 'ISO-8859-1' + ), + )) +def test_get_encoding_from_headers(value, expected): + assert get_encoding_from_headers(value) == expected + + +@pytest.mark.parametrize( + 'value, length', ( + ('', 0), + ('T', 1), + ('Test', 4), + )) +def test_iter_slices(value, length): + assert len(list(iter_slices(value, 1))) == length + + +@pytest.mark.parametrize( + 'value, expected', ( + ( + '; rel=front; type="image/jpeg"', + [{'url': 'http:/.../front.jpeg', 'rel': 'front', 'type': 'image/jpeg'}] + ), + ( + '', + [{'url': 'http:/.../front.jpeg'}] + ), + ( + ';', + [{'url': 'http:/.../front.jpeg'}] + ), + ( + '; type="image/jpeg",;', + [ + {'url': 'http:/.../front.jpeg', 'type': 'image/jpeg'}, + {'url': 'http://.../back.jpeg'} + ] + ), + )) +def test_parse_header_links(value, expected): + assert parse_header_links(value) == expected + + +@pytest.mark.parametrize( + 'value, expected', ( + ('example.com/path', 'http://example.com/path'), + ('//example.com/path', 'http://example.com/path'), + )) +def test_prepend_scheme_if_needed(value, expected): + assert prepend_scheme_if_needed(value, 'http') == expected + + +@pytest.mark.parametrize( + 'value, expected', ( + ('T', 'T'), + (b'T', 'T'), + (u'T', 'T'), + )) +def test_to_native_string(value, expected): + assert to_native_string(value) == expected + + +@pytest.mark.parametrize( + 'url, expected', ( + ('http://u:p@example.com/path?a=1#test', 'http://example.com/path?a=1'), + ('http://example.com/path', 'http://example.com/path'), + ('//u:p@example.com/path', '//example.com/path'), + ('//example.com/path', '//example.com/path'), + ('example.com/path', '//example.com/path'), + ('scheme:u:p@example.com/path', 'scheme://example.com/path'), + )) +def test_urldefragauth(url, expected): + assert urldefragauth(url) == expected + + +@pytest.mark.parametrize( + 'url, expected', ( + ('http://192.168.0.1:5000/', True), + ('http://192.168.0.1/', True), + ('http://172.16.1.1/', True), + ('http://172.16.1.1:5000/', True), + ('http://localhost.localdomain:5000/v1.0/', True), + ('http://172.16.1.12/', False), + ('http://172.16.1.12:5000/', False), + ('http://google.com:5000/v1.0/', False), + )) +def test_should_bypass_proxies(url, expected, monkeypatch): + """ + Tests for function should_bypass_proxies to check if proxy can be bypassed or not + """ + monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') + monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') + assert should_bypass_proxies(url) == expected diff --git a/tests/testserver/__init__.py b/tests/testserver/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/testserver/server.py b/tests/testserver/server.py new file mode 100644 index 00000000..7a92c87d --- /dev/null +++ b/tests/testserver/server.py @@ -0,0 +1,125 @@ +import threading +import socket +import select + + +def consume_socket_content(sock, timeout=0.5): + chunks = 65536 + content = b'' + more_to_read = select.select([sock], [], [], timeout)[0] + + while more_to_read: + new_content = sock.recv(chunks) + + if not new_content: + break + + content += new_content + # stop reading if no new data is received for a while + more_to_read = select.select([sock], [], [], timeout)[0] + + return content + + +class Server(threading.Thread): + """Dummy server using for unit testing""" + WAIT_EVENT_TIMEOUT = 5 + + def __init__(self, handler=None, host='localhost', port=0, requests_to_handle=1, wait_to_close_event=None): + super(Server, self).__init__() + + self.handler = handler or consume_socket_content + self.handler_results = [] + + self.host = host + self.port = port + self.requests_to_handle = requests_to_handle + + self.wait_to_close_event = wait_to_close_event + self.ready_event = threading.Event() + self.stop_event = threading.Event() + + @classmethod + def text_response_server(cls, text, request_timeout=0.5, **kwargs): + def text_response_handler(sock): + request_content = consume_socket_content(sock, timeout=request_timeout) + sock.send(text.encode('utf-8')) + + return request_content + + + return Server(text_response_handler, **kwargs) + + @classmethod + def basic_response_server(cls, **kwargs): + return cls.text_response_server( + "HTTP/1.1 200 OK\r\n" + + "Content-Length: 0\r\n\r\n", + **kwargs + ) + + def run(self): + try: + self.server_sock = self._create_socket_and_bind() + # in case self.port = 0 + self.port = self.server_sock.getsockname()[1] + self.ready_event.set() + self._handle_requests() + + if self.wait_to_close_event: + self.wait_to_close_event.wait(self.WAIT_EVENT_TIMEOUT) + finally: + self.ready_event.set() # just in case of exception + self._close_server_sock_ignore_errors() + self.stop_event.set() + + def _create_socket_and_bind(self): + sock = socket.socket() + sock.bind((self.host, self.port)) + sock.listen(0) + return sock + + def _close_server_sock_ignore_errors(self): + try: + self.server_sock.close() + except IOError: + pass + + def _handle_requests(self): + for _ in range(self.requests_to_handle): + sock = self._accept_connection() + if not sock: + break + + handler_result = self.handler(sock) + + self.handler_results.append(handler_result) + + def _accept_connection(self): + try: + ready, _, _ = select.select([self.server_sock], [], [], self.WAIT_EVENT_TIMEOUT) + if not ready: + return None + + return self.server_sock.accept()[0] + except (select.error, socket.error): + return None + + def __enter__(self): + self.start() + self.ready_event.wait(self.WAIT_EVENT_TIMEOUT) + return self.host, self.port + + def __exit__(self, exc_type, exc_value, traceback): + if exc_type is None: + self.stop_event.wait(self.WAIT_EVENT_TIMEOUT) + else: + if self.wait_to_close_event: + # avoid server from waiting for event timeouts + # if an exception is found in the main thread + self.wait_to_close_event.set() + + # ensure server thread doesn't get stuck waiting for connections + self._close_server_sock_ignore_errors() + self.join() + return False # allow exceptions to propagate diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 00000000..6cb75bfb --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,17 @@ +import contextlib +import os + + +@contextlib.contextmanager +def override_environ(**kwargs): + save_env = dict(os.environ) + for key, value in kwargs.items(): + if value is None: + del os.environ[key] + else: + os.environ[key] = value + try: + yield + finally: + os.environ.clear() + os.environ.update(save_env)