diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..e9b51ab1 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[run] +omit = requests/packages/* \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..0887ad29 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,18 @@ +language: python +python: + - "2.6" + - "2.7" + - "3.3" + - "3.4" + - "3.5" + - "3.6" + - "3.7-dev" + # - "pypy" -- appears to hang + # - "pypy3" +# command to install dependencies +install: "make" +# command to run tests +script: + - make coverage +after_success: + - pipenv run codecov diff --git a/AUTHORS.rst b/AUTHORS.rst index 93a8a979..6c2e39c5 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -177,3 +177,6 @@ Patches and Suggestions - Philipp Konrad (`@gardiac2002 `_) - Hussain Tamboli (`@hussaintamboli `_) - Casey Davidson (`@davidsoncasey `_) +- Andrii Soldatenko (`@a_soldatenko `_) +- Moinuddin Quadri (`@moin18 `_) +- Matt Kohl (`@mattkohl `_) diff --git a/HISTORY.rst b/HISTORY.rst index 01ec2dd8..5ec8bb21 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,36 @@ Release History --------------- +2.13.0 (2017-01-24) ++++++++++++++++++++ + +**Features** + +- Only load the ``idna`` library when we've determined we need it. This will + save some memory for users. + +**Miscellaneous** + +- Updated bundled urllib3 to 1.20. +- Updated bundled idna to 2.2. + +2.12.5 (2017-01-18) ++++++++++++++++++++ + +**Bugfixes** + +- Fixed an issue with JSON encoding detection, specifically detecting + big-endian UTF-32 with BOM. + +2.12.4 (2016-12-14) ++++++++++++++++++++ + +**Bugfixes** + +- Fixed regression from 2.12.2 where non-string types were rejected in the + basic auth parameters. While support for this behaviour has been readded, + the behaviour is deprecated and will be removed in the future. + 2.12.3 (2016-12-01) +++++++++++++++++++ diff --git a/LICENSE b/LICENSE index 8c1dd448..db78ea69 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2016 Kenneth Reitz +Copyright 2017 Kenneth Reitz Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/MANIFEST.in b/MANIFEST.in index 439de496..46ef0b46 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include README.rst LICENSE NOTICE HISTORY.rst test_requests.py requirements.txt requests/cacert.pem +include README.rst LICENSE NOTICE HISTORY.rst test_requests.py requests/cacert.pem diff --git a/Makefile b/Makefile index d74b5aa1..ee737f9e 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,17 @@ .PHONY: docs init: - pip install -r requirements.txt + pip install pipenv + pipenv lock + pipenv install --dev test: # This runs all of the tests. To run an individual test, run py.test with # the -k flag, like "py.test -k test_path_is_not_double_encoded" - py.test tests + pipenv run py.test tests coverage: - py.test --verbose --cov-report term --cov=requests tests - -ci: init - py.test --junitxml=junit.xml + pipenv run py.test --cov-config .coveragerc --verbose --cov-report term --cov-report xml --cov=requests tests certs: curl http://ci.kennethreitz.org/job/ca-bundle/lastSuccessfulBuild/artifact/cacerts.pem -o requests/cacert.pem @@ -20,7 +19,7 @@ certs: deps: urllib3 chardet idna urllib3: - git clone https://github.com/shazow/urllib3.git && \ + git clone -b release https://github.com/shazow/urllib3.git && \ rm -fr requests/packages/urllib3 && \ cd urllib3 && \ git checkout `git describe --abbrev=0 --tags` && \ @@ -44,19 +43,15 @@ idna: git checkout `git describe --abbrev=0 --tags` && \ cd .. && \ mv idna/idna requests/packages/ && \ - find requests/packages/idna -type f -exec sed -i "" "s/^from idna/from /" {} \; && \ rm -fr idna publish: - python setup.py register - python setup.py sdist upload - python setup.py bdist_wheel --universal upload + pip install 'twine>=1.5.0' + python setup.py sdist + python setup.py bdist_wheel --universal + twine upload dist/* rm -fr build dist .egg requests.egg-info - -docs-init: - pip install -r docs/requirements.txt - docs: cd docs && make html @echo "\033[95m\n\nBuild successful! View the docs homepage at docs/_build/html/index.html.\n\033[0m" diff --git a/Pipfile b/Pipfile new file mode 100644 index 00000000..617c7ffa --- /dev/null +++ b/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true + +[dev-packages] +pytest = ">=2.8.0" +codecov = "*" +pytest-httpbin = "==0.0.7" +sphinx = "*" +pytest-mock = "*" +pytest-cov = "*" +pysocks = "*" +alabaster = "*" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 00000000..d522ed2f --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,153 @@ +{ + "default": {}, + "develop": { + "snowballstemmer": { + "version": "==1.2.1", + "hash": "sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89" + }, + "Werkzeug": { + "version": "==0.11.15", + "hash": "sha256:c6f6f89124df0514d886782c658c3e12f2caaa94da34cee3fd82eebf4ebf052b" + }, + "six": { + "version": "==1.10.0", + "hash": "sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1" + }, + "funcsigs": { + "version": "==1.0.2", + "hash": "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca" + }, + "coverage": { + "version": "==4.3.4", + "hash": "sha256:36407249a0b6669c6ad4425b0f29685579df745480c03afa70f101f09f4eead3" + }, + "Flask": { + "version": "==0.12", + "hash": "sha256:7f03bb2c255452444f7265eddb51601806e5447b6f8a2d50bbc77a654a14c118" + }, + "alabaster": { + "version": "==0.7.9", + "hash": "sha256:d3e64a74919373d6d4d1d36bd717206584cb64cbb0532dfce3bc2081cba6817b" + }, + "pytest-mock": { + "version": "==1.5.0", + "hash": "sha256:8e0fd43280c717f36920b60356bd713291b81a61704c94bc13aae9a12ef7fbd8" + }, + "packaging": { + "version": "==16.8", + "hash": "sha256:99276dc6e3a7851f32027a68f1095cd3f77c148091b092ea867a351811cfe388" + }, + "MarkupSafe": { + "version": "==0.23", + "hash": "sha256:a4ec1aff59b95a14b45eb2e23761a0179e98319da5a7eb76b56ea8cdc7b871c3" + }, + "pytz": { + "version": "==2016.10", + "hash": "sha256:a1ea35e87a63c7825846d5b5c81d23d668e8a102d3b1b465ce95afe1b3a2e065" + }, + "codecov": { + "version": "==2.0.5", + "hash": "sha256:9fb0cd4a43fe538b4ea229607d0a7d65b00f9bfb37bb6af60a17f4ac33707334" + }, + "pytest-httpbin": { + "version": "==0.2.3", + "hash": "sha256:c5b698dfa474ffc9caebcb35e34346b753eb226aea5c2e1b69fefedbcf161bf8" + }, + "httpbin": { + "version": "==0.5.0", + "hash": "sha256:710069973216d4bbf9ab6757f1e9a1f3be05832ce77da023adce0a98dfeecfee" + }, + "pyparsing": { + "version": "==2.1.10", + "hash": "sha256:67101d7acee692962f33dd30b5dce079ff532dd9aa99ff48d52a3dad51d2fe84" + }, + "click": { + "version": "==6.7", + "hash": "sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d" + }, + "appdirs": { + "version": "==1.4.0", + "hash": "sha256:85e58578db8f29538f3109c11250c2a5514a2fcdc9890d9b2fe777eb55517736" + }, + "imagesize": { + "version": "==0.7.1", + "hash": "sha256:6ebdc9e0ad188f9d1b2cdd9bc59cbe42bf931875e829e7a595e6b3abdc05cdfb" + }, + "argparse": { + "version": "==1.4.0", + "hash": "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" + }, + "sphinx": { + "version": "==1.5.2", + "hash": "sha256:57c8636e1d23f6c01fb19911a8f255f1b4934ba69feb55bd4dd0f097ebb04f05" + }, + "pbr": { + "version": "==1.10.0", + "hash": "sha256:f5cf7265a80636ecff66806d13494cbf9d77a3758a65fd8b4d4d4bee81b0c375" + }, + "babel": { + "version": "==2.3.4", + "hash": "sha256:3318ed2960240d61cbc6558858ee00c10eed77a6508c4d1ed8e6f7f48399c975" + }, + "py": { + "version": "==1.4.32", + "hash": "sha256:2d4bba2e25fff58140e6bdce1e485e89bb59776adbe01d490baa6b1f37a3dd6b" + }, + "pytest-cov": { + "version": "==2.4.0", + "hash": "sha256:10e37e876f49ddec80d6c83a54b657157f1387ebc0f7755285f8c156130014a1" + }, + "pytest": { + "version": "==3.0.6", + "hash": "sha256:da0ab50c7eec0683bc24f1c1137db1f4111752054ecdad63125e7ec71316b813" + }, + "docutils": { + "version": "==0.13.1", + "hash": "sha256:de454f1015958450b72641165c08afe7023cd7e3944396448f2fb1b0ccba9d77" + }, + "Pygments": { + "version": "==2.2.0", + "hash": "sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d" + }, + "Jinja2": { + "version": "==2.9.5", + "hash": "sha256:a7b7438120dbe76a8e735ef7eba6048eaf4e0b7dbc530e100812f8ec462a4d50" + }, + "decorator": { + "version": "==4.0.11", + "hash": "sha256:73cbaadb8bc4e3c65fe1100773d56331a2d756cc0f5c7b9d8d5d5223fe04f600" + }, + "setuptools": { + "version": "==34.1.0", + "hash": "sha256:edd9d39782fe38b9c533002b2e6fdf06498793cbd29266accdcc519431d4b7ba" + }, + "requests": { + "version": "==2.13.0", + "hash": "sha256:1a720e8862a41aa22e339373b526f508ef0c8988baf48b84d3fc891a8e237efb" + }, + "itsdangerous": { + "version": "==0.24", + "hash": "sha256:cbb3fcf8d3e33df861709ecaf89d9e6629cff0a217bc2848f1b41cd30d360519" + }, + "pysocks": { + "version": "==1.6.6", + "hash": "sha256:02419a225ff5dcfc3c9695ef8fc9b4d8cc99658e650c6d4718d4c8f451e63f41" + }, + "mock": { + "version": "==2.0.0", + "hash": "sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1" + } + }, + "_meta": { + "sources": [ + { + "url": "https://pypi.python.org/simple", + "verify_ssl": true + } + ], + "requires": {}, + "hash": { + "sha256": "0b4728fe74b683054ddde5b9dba7dd674ce17b6726764407a9779b4fdd0afd47" + } + } +} \ No newline at end of file diff --git a/README.rst b/README.rst index 57a4f334..d276e42a 100644 --- a/README.rst +++ b/README.rst @@ -3,7 +3,28 @@ Requests: HTTP for Humans .. image:: https://img.shields.io/pypi/v/requests.svg :target: https://pypi.python.org/pypi/requests - + +.. image:: https://img.shields.io/pypi/l/requests.svg + :target: https://pypi.python.org/pypi/requests + +.. image:: https://img.shields.io/pypi/wheel/requests.svg + :target: https://pypi.python.org/pypi/requests + +.. image:: https://img.shields.io/pypi/pyversions/requests.svg + :target: https://pypi.python.org/pypi/requests + +.. image:: https://travis-ci.org/kennethreitz/requests.svg?branch=master + :target: https://travis-ci.org/kennethreitz/requests + +.. image:: https://codecov.io/github/kennethreitz/requests/coverage.svg?branch=master + :target: https://codecov.io/github/kennethreitz/requests + :alt: codecov.io + +.. image:: https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg + :target: https://saythanks.io/to/kennethreitz + + + Requests is the only *Non-GMO* HTTP library for Python, safe for human consumption. @@ -40,7 +61,7 @@ are 100% automatic, powered by `urllib3 `_, which is embedded within Requests. Besides, all the cool kids are doing it. Requests is one of the most -downloaded Python packages of all time, pulling in over 7,000,000 downloads +downloaded Python packages of all time, pulling in over 11,000,000 downloads every month. You don't want to be left out! Feature Support @@ -65,7 +86,7 @@ Requests is ready for today's web. - Chunked Requests - Thread-safety -Requests officially supports Python 2.6–2.7 & 3.3–3.5, and runs great on PyPy. +Requests officially supports Python 2.6–2.7 & 3.3–3.7, and runs great on PyPy. Installation ------------ diff --git a/docs/.api.rst.swo b/docs/.api.rst.swo deleted file mode 100644 index 76d80e7d..00000000 Binary files a/docs/.api.rst.swo and /dev/null differ diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html index 3ddd01ff..457688d9 100644 --- a/docs/_templates/sidebarintro.html +++ b/docs/_templates/sidebarintro.html @@ -28,13 +28,14 @@

More Kenneth Reitz projects:

diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index b9cc5991..798f8091 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -32,13 +32,14 @@

More Kenneth Reitz projects:

diff --git a/docs/community/faq.rst b/docs/community/faq.rst index c87687af..e835b122 100644 --- a/docs/community/faq.rst +++ b/docs/community/faq.rst @@ -59,6 +59,7 @@ supported: * Python 3.3 * Python 3.4 * Python 3.5 +* Python 3.6 * PyPy What are "hostname doesn't match" errors? diff --git a/docs/conf.py b/docs/conf.py index fb8a01e0..41bc6066 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -59,7 +59,7 @@ master_doc = 'index' # General information about the project. project = u'Requests' -copyright = u'2016. A Kenneth Reitz Project' +copyright = u'2017. A Kenneth Reitz Project' author = u'Kenneth Reitz' # The version info for the project you're documenting, acts as replacement for diff --git a/docs/dev/todo.rst b/docs/dev/todo.rst index 79b95a21..d4862fbb 100644 --- a/docs/dev/todo.rst +++ b/docs/dev/todo.rst @@ -23,13 +23,23 @@ Development Dependencies You'll need to install py.test in order to run the Requests' test suite:: - $ pip install -r requirements.txt - $ py.test - platform darwin -- Python 2.7.3 -- pytest-2.3.4 - collected 25 items + $ pip install pipenv + $ pipenv lock + $ pipenv install --dev + $ pipenv run py.test tests + ============================= test session starts ============================== + platform darwin -- Python 3.4.4, pytest-3.0.6, py-1.4.32, pluggy-0.4.0 + ... + collected 445 items - test_requests.py ......................... - 25 passed in 3.50 seconds + tests/test_hooks.py ... + tests/test_lowlevel.py ............ + tests/test_requests.py ........................................................... + tests/test_structures.py .................... + tests/test_testserver.py ........... + tests/test_utils.py ..s........................................................... + + ============== 442 passed, 1 skipped, 2 xpassed in 46.48 seconds =============== Runtime Environments -------------------- @@ -41,6 +51,7 @@ Requests currently supports the following versions of Python: - Python 3.3 - Python 3.4 - Python 3.5 +- Python 3.6 - PyPy Google AppEngine is not officially supported although support is available diff --git a/docs/index.rst b/docs/index.rst index d8279a9c..c1d78c87 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,14 +8,36 @@ Requests: HTTP for Humans Release v\ |version|. (:ref:`Installation `) -Requests is the only *Non-GMO* HTTP library for Python, safe for human +.. image:: https://img.shields.io/pypi/l/requests.svg + :target: https://pypi.python.org/pypi/requests + +.. image:: https://img.shields.io/pypi/wheel/requests.svg + :target: https://pypi.python.org/pypi/requests + +.. image:: https://img.shields.io/pypi/pyversions/requests.svg + :target: https://pypi.python.org/pypi/requests + +.. image:: https://travis-ci.org/kennethreitz/requests.svg?branch=master + :target: https://travis-ci.org/kennethreitz/requests + +.. image:: https://codecov.io/github/kennethreitz/requests/coverage.svg?branch=master + :target: https://codecov.io/github/kennethreitz/requests + :alt: codecov.io + +.. image:: https://img.shields.io/badge/Say%20Thanks!-🦉-1EAEDB.svg + :target: https://saythanks.io/to/kennethreitz + + +**Requests** is the only *Non-GMO* HTTP library for Python, safe for human consumption. -**Warning:** Recreational use of other HTTP libraries may result in dangerous side-effects, +*Warning: Recreational use of other HTTP libraries may result in dangerous side-effects, including: security vulnerabilities, verbose code, reinventing the wheel, -constantly reading documentation, depression, headaches, or even death. +constantly reading documentation, depression, headaches, or even death.* -Behold, the power of Requests:: +------------------- + +**Behold, the power of Requests**:: >>> r = requests.get('https://api.github.com/user', auth=('user', 'pass')) >>> r.status_code @@ -32,7 +54,7 @@ Behold, the power of Requests:: See `similar code, sans Requests `_. -Requests allows you to send *organic, grass-fed* HTTP/1.1 requests, without the +**Requests** allows you to send *organic, grass-fed* HTTP/1.1 requests, without the need for manual labor. There's no need to manually add query strings to your URLs, or to form-encode your POST data. Keep-alive and HTTP connection pooling are 100% automatic, powered by `urllib3 `_, @@ -43,7 +65,7 @@ User Testimonials The NSA, Her Majesty's Government, Amazon, Google, Twilio, Runscope, Mozilla, Heroku, PayPal, NPR, Obama for America, Transifex, Native Instruments, The Washington -Post, Twitter, SoundCloud, Kippt, Readability, Sony, and Federal U.S. +Post, Twitter, SoundCloud, Kippt, Sony, and Federal U.S. Institutions that prefer to be unnamed claim to use Requests internally. **Armin Ronacher** @@ -51,43 +73,43 @@ Institutions that prefer to be unnamed claim to use Requests internally. right level of abstraction. **Matt DeBoard** - I'm going to get @kennethreitz's Python requests module tattooed + I'm going to get `@kennethreitz `_'s Python requests module tattooed on my body, somehow. The whole thing. **Daniel Greenfeld** Nuked a 1200 LOC spaghetti code library with 10 lines of code thanks to - @kennethreitz's request library. Today has been AWESOME. + `@kennethreitz `_'s request library. Today has been AWESOME. **Kenny Meyers** Python HTTP: When in doubt, or when not in doubt, use Requests. Beautiful, simple, Pythonic. Requests is one of the most downloaded Python packages of all time, pulling in -over 7,000,000 downloads every month. All the cool kids are doing it! +over 11,000,000 downloads every month. All the cool kids are doing it! -Supported Features ------------------- +Beloved Features +---------------- Requests is ready for today's web. -- International Domains and URLs - Keep-Alive & Connection Pooling +- International Domains and URLs - Sessions with Cookie Persistence - Browser-style SSL Verification +- Automatic Content Decoding - Basic/Digest Authentication - Elegant Key/Value Cookies - Automatic Decompression -- Automatic Content Decoding - Unicode Response Bodies -- Multipart File Uploads - HTTP(S) Proxy Support -- Connection Timeouts +- Multipart File Uploads - Streaming Downloads -- ``.netrc`` Support +- Connection Timeouts - Chunked Requests +- ``.netrc`` Support - Thread-safety -Requests officially supports Python 2.6–2.7 & 3.3–3.5, and runs great on PyPy. +Requests officially supports Python 2.6–2.7 & 3.3–3.7, and runs great on PyPy. The User Guide diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 9a404908..3c63394d 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -220,7 +220,7 @@ This list of trusted CAs can also be specified through the ``REQUESTS_CA_BUNDLE` Requests can also ignore verifying the SSL certificate if you set ``verify`` to False:: - >>> requests.get('https://kennethreitz.com', verify=False) + >>> requests.get('https://kennethreitz.org', verify=False) By default, ``verify`` is set to True. Option ``verify`` only applies to host certs. @@ -229,7 +229,7 @@ You can also specify a local cert to use as client side certificate, as a single file (containing the private key and the certificate) or as a tuple of both file's path:: - >>> requests.get('https://kennethreitz.com', cert=('/path/client.cert', '/path/client.key')) + >>> requests.get('https://kennethreitz.org', cert=('/path/client.cert', '/path/client.key')) or persistent:: @@ -239,7 +239,7 @@ or persistent:: If you specify a wrong path or an invalid cert, you'll get a SSLError:: - >>> requests.get('https://kennethreitz.com', cert='/wrong_path/client.pem') + >>> requests.get('https://kennethreitz.org', cert='/wrong_path/client.pem') SSLError: [Errno 336265225] _ssl.c:347: error:140B0009:SSL routines:SSL_CTX_use_PrivateKey_file:PEM lib .. warning:: The private key to your local certificate *must* be unencrypted. diff --git a/docs/user/authentication.rst b/docs/user/authentication.rst index 58175623..2b7a8690 100644 --- a/docs/user/authentication.rst +++ b/docs/user/authentication.rst @@ -60,7 +60,7 @@ OAuth 1 Authentication ---------------------- A common form of authentication for several web APIs is OAuth. The ``requests-oauthlib`` -library allows Requests users to easily make OAuth authenticated requests:: +library allows Requests users to easily make OAuth 1 authenticated requests:: >>> import requests >>> from requests_oauthlib import OAuth1 @@ -76,6 +76,17 @@ For more information on how to OAuth flow works, please see the official `OAuth` For examples and documentation on requests-oauthlib, please see the `requests_oauthlib`_ repository on GitHub +OAuth 2 and OpenID Connect Authentication +----------------------------------------- + +The ``requests-oauthlib`` library also handles OAuth 2, the authentication mechanism +underpinning OpenID Connect. See the `requests-oauthlib OAuth2 documentation`_ for +details of the various OAuth 2 credential management flows: + +* `Web Application Flow`_ +* `Mobile Application Flow`_ +* `Legacy Application Flow`_ +* `Backend Application Flow`_ Other Authentication -------------------- @@ -123,6 +134,11 @@ Further examples can be found under the `Requests organization`_ and in the .. _OAuth: http://oauth.net/ .. _requests_oauthlib: https://github.com/requests/requests-oauthlib +.. _requests-oauthlib OAuth2 documentation: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html +.. _Web Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#web-application-flow +.. _Mobile Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#mobile-application-flow +.. _Legacy Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#legacy-application-flow +.. _Backend Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#backend-application-flow .. _Kerberos: https://github.com/requests/requests-kerberos .. _NTLM: https://github.com/requests/requests-ntlm .. _Requests organization: https://github.com/requests diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 232760f4..60cc73fb 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -57,8 +57,8 @@ Passing Parameters In URLs You often want to send some sort of data in the URL's query string. If you were constructing the URL by hand, this data would be given as key/value pairs in the URL after a question mark, e.g. ``httpbin.org/get?key=val``. -Requests allows you to provide these arguments as a dictionary, using the -``params`` keyword argument. As an example, if you wanted to pass +Requests allows you to provide these arguments as a dictionary of strings, +using the ``params`` keyword argument. As an example, if you wanted to pass ``key1=value1`` and ``key2=value2`` to ``httpbin.org/get``, you would use the following code:: diff --git a/requests/api.py b/requests/api.py index 0e25ded8..375ac2e2 100644 --- a/requests/api.py +++ b/requests/api.py @@ -34,7 +34,7 @@ def request(method, url, session=None, **kwargs): before giving up, as a float, or a :ref:`(connect timeout, read timeout) ` tuple. :type timeout: float or tuple - :param allow_redirects: (optional) Boolean. Enable/disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. + :param allow_redirects: (optional) Boolean. Enable/disable GET/OPTIONS/POST/PUT/PATCH/DELETE/HEAD redirection. Defaults to ``True``. :type allow_redirects: bool :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. :param verify: (optional) whether the SSL cert will be verified. A CA_BUNDLE path can also be provided. Defaults to ``True``. diff --git a/requests/auth.py b/requests/auth.py index 04d96ba4..c2eb2db9 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -198,6 +198,12 @@ class HTTPDigestAuth(AuthBase): :rtype: requests.Response """ + # If response is not 4xx, do not auth + # See https://github.com/kennethreitz/requests/issues/3772 + if not 400 <= r.status_code < 500: + self._thread_local.num_401_calls = 1 + return r + if self._thread_local.pos is not None: # Rewind the file position indicator of the body to where # it was to resend the request. diff --git a/requests/models.py b/requests/models.py index cd07c3d8..0b3838b2 100644 --- a/requests/models.py +++ b/requests/models.py @@ -10,6 +10,7 @@ This module contains the primary objects that power Requests. import collections import datetime import codecs +import sys # Import encoding now, to avoid implicit import later. # Implicit import within threads may cause LookupError when standard library is in a ZIP, @@ -23,7 +24,6 @@ from .structures import CaseInsensitiveDict import requests from .auth import HTTPBasicAuth from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar -from .packages import idna from .packages.urllib3.fields import RequestField from .packages.urllib3.filepost import encode_multipart_formdata from .packages.urllib3.util import parse_url @@ -335,6 +335,22 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): raise ValueError('Request method cannot be "None"') self.method = to_native_string(self.method).upper() + @staticmethod + def _get_idna_encoded_host(host): + try: + from .packages import idna + except ImportError: + # tolerate the possibility of downstream repackagers unvendoring `requests` + # For more information, read: packages/__init__.py + import idna + sys.modules['requests.packages.idna'] = idna + + try: + host = idna.encode(host, uts46=True).decode('utf-8') + except idna.IDNAError: + raise UnicodeError + return host + def prepare_url(self, url, params): """Prepares the given HTTP URL.""" #: Accept objects that have string representations. @@ -372,17 +388,17 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): if not host: raise InvalidURL("Invalid URL %r: No host supplied" % url) - # In general, we want to try IDNA encoding every hostname, as that - # allows users to automatically get the correct behaviour. However, - # we’re quite strict about IDNA encoding, so certain valid hostnames - # may fail to encode. On failure, we verify the hostname meets a - # minimum standard of only containing ASCII characters, and not starting - # with a wildcard (*), before allowing the unencoded hostname through. - try: - host = idna.encode(host, uts46=True).decode('utf-8') - except (UnicodeError, idna.IDNAError): - if not unicode_is_ascii(host) or host.startswith(u'*'): + # In general, we want to try IDNA encoding the hostname if the string contains + # non-ASCII characters. This allows users to automatically get the correct IDNA + # behaviour. For strings containing only ASCII characters, we need to also verify + # it doesn't start with a wildcard (*), before allowing the unencoded hostname. + if not unicode_is_ascii(host): + try: + host = self._get_idna_encoded_host(host) + except UnicodeError: raise InvalidURL('URL has an invalid label.') + elif host.startswith(u'*'): + raise InvalidURL('URL has an invalid label.') # Carefully reconstruct the network location netloc = auth or '' diff --git a/requests/packages/__init__.py b/requests/packages/__init__.py index 4077265e..971c2ad0 100644 --- a/requests/packages/__init__.py +++ b/requests/packages/__init__.py @@ -34,9 +34,3 @@ try: except ImportError: import chardet sys.modules['%s.chardet' % __name__] = chardet - -try: - from . import idna -except ImportError: - import idna - sys.modules['%s.idna' % __name__] = idna diff --git a/requests/packages/idna/uts46data.py b/requests/packages/idna/uts46data.py index 64e2c687..48da8408 100644 --- a/requests/packages/idna/uts46data.py +++ b/requests/packages/idna/uts46data.py @@ -3,7 +3,9 @@ """IDNA Mapping Table from UTS46.""" -uts46data = ( + +def _seg_0(): + return [ (0x0, '3'), (0x1, '3'), (0x2, '3'), @@ -104,6 +106,10 @@ uts46data = ( (0x61, 'V'), (0x62, 'V'), (0x63, 'V'), + ] + +def _seg_1(): + return [ (0x64, 'V'), (0x65, 'V'), (0x66, 'V'), @@ -204,6 +210,10 @@ uts46data = ( (0xC5, 'M', u'å'), (0xC6, 'M', u'æ'), (0xC7, 'M', u'ç'), + ] + +def _seg_2(): + return [ (0xC8, 'M', u'è'), (0xC9, 'M', u'é'), (0xCA, 'M', u'ê'), @@ -304,6 +314,10 @@ uts46data = ( (0x129, 'V'), (0x12A, 'M', u'ī'), (0x12B, 'V'), + ] + +def _seg_3(): + return [ (0x12C, 'M', u'ĭ'), (0x12D, 'V'), (0x12E, 'M', u'į'), @@ -404,6 +418,10 @@ uts46data = ( (0x191, 'M', u'ƒ'), (0x192, 'V'), (0x193, 'M', u'ɠ'), + ] + +def _seg_4(): + return [ (0x194, 'M', u'ɣ'), (0x195, 'V'), (0x196, 'M', u'ɩ'), @@ -504,6 +522,10 @@ uts46data = ( (0x20A, 'M', u'ȋ'), (0x20B, 'V'), (0x20C, 'M', u'ȍ'), + ] + +def _seg_5(): + return [ (0x20D, 'V'), (0x20E, 'M', u'ȏ'), (0x20F, 'V'), @@ -604,6 +626,10 @@ uts46data = ( (0x375, 'V'), (0x376, 'M', u'ͷ'), (0x377, 'V'), + ] + +def _seg_6(): + return [ (0x378, 'X'), (0x37A, '3', u' ι'), (0x37B, 'V'), @@ -704,6 +730,10 @@ uts46data = ( (0x401, 'M', u'ё'), (0x402, 'M', u'ђ'), (0x403, 'M', u'ѓ'), + ] + +def _seg_7(): + return [ (0x404, 'M', u'є'), (0x405, 'M', u'ѕ'), (0x406, 'M', u'і'), @@ -804,6 +834,10 @@ uts46data = ( (0x49C, 'M', u'ҝ'), (0x49D, 'V'), (0x49E, 'M', u'ҟ'), + ] + +def _seg_8(): + return [ (0x49F, 'V'), (0x4A0, 'M', u'ҡ'), (0x4A1, 'V'), @@ -904,6 +938,10 @@ uts46data = ( (0x501, 'V'), (0x502, 'M', u'ԃ'), (0x503, 'V'), + ] + +def _seg_9(): + return [ (0x504, 'M', u'ԅ'), (0x505, 'V'), (0x506, 'M', u'ԇ'), @@ -1004,6 +1042,10 @@ uts46data = ( (0x678, 'M', u'يٴ'), (0x679, 'V'), (0x6DD, 'X'), + ] + +def _seg_10(): + return [ (0x6DE, 'V'), (0x70E, 'X'), (0x710, 'V'), @@ -1104,6 +1146,10 @@ uts46data = ( (0xA5D, 'X'), (0xA5E, 'M', u'ਫ਼'), (0xA5F, 'X'), + ] + +def _seg_11(): + return [ (0xA66, 'V'), (0xA76, 'X'), (0xA81, 'V'), @@ -1204,6 +1250,10 @@ uts46data = ( (0xC2A, 'V'), (0xC34, 'X'), (0xC35, 'V'), + ] + +def _seg_12(): + return [ (0xC3A, 'X'), (0xC3D, 'V'), (0xC45, 'X'), @@ -1304,6 +1354,10 @@ uts46data = ( (0xE84, 'V'), (0xE85, 'X'), (0xE87, 'V'), + ] + +def _seg_13(): + return [ (0xE89, 'X'), (0xE8A, 'V'), (0xE8B, 'X'), @@ -1404,6 +1458,10 @@ uts46data = ( (0x1250, 'V'), (0x1257, 'X'), (0x1258, 'V'), + ] + +def _seg_14(): + return [ (0x1259, 'X'), (0x125A, 'V'), (0x125E, 'X'), @@ -1504,6 +1562,10 @@ uts46data = ( (0x1A8A, 'X'), (0x1A90, 'V'), (0x1A9A, 'X'), + ] + +def _seg_15(): + return [ (0x1AA0, 'V'), (0x1AAE, 'X'), (0x1B00, 'V'), @@ -1604,6 +1666,10 @@ uts46data = ( (0x1DA7, 'M', u'ᵻ'), (0x1DA8, 'M', u'ʝ'), (0x1DA9, 'M', u'ɭ'), + ] + +def _seg_16(): + return [ (0x1DAA, 'M', u'ᶅ'), (0x1DAB, 'M', u'ʟ'), (0x1DAC, 'M', u'ɱ'), @@ -1704,6 +1770,10 @@ uts46data = ( (0x1E48, 'M', u'ṉ'), (0x1E49, 'V'), (0x1E4A, 'M', u'ṋ'), + ] + +def _seg_17(): + return [ (0x1E4B, 'V'), (0x1E4C, 'M', u'ṍ'), (0x1E4D, 'V'), @@ -1804,6 +1874,10 @@ uts46data = ( (0x1EB1, 'V'), (0x1EB2, 'M', u'ẳ'), (0x1EB3, 'V'), + ] + +def _seg_18(): + return [ (0x1EB4, 'M', u'ẵ'), (0x1EB5, 'V'), (0x1EB6, 'M', u'ặ'), @@ -1904,6 +1978,10 @@ uts46data = ( (0x1F2B, 'M', u'ἣ'), (0x1F2C, 'M', u'ἤ'), (0x1F2D, 'M', u'ἥ'), + ] + +def _seg_19(): + return [ (0x1F2E, 'M', u'ἦ'), (0x1F2F, 'M', u'ἧ'), (0x1F30, 'V'), @@ -2004,6 +2082,10 @@ uts46data = ( (0x1FAC, 'M', u'ὤι'), (0x1FAD, 'M', u'ὥι'), (0x1FAE, 'M', u'ὦι'), + ] + +def _seg_20(): + return [ (0x1FAF, 'M', u'ὧι'), (0x1FB0, 'V'), (0x1FB2, 'M', u'ὰι'), @@ -2104,6 +2186,10 @@ uts46data = ( (0x204A, 'V'), (0x2057, 'M', u'′′′′'), (0x2058, 'V'), + ] + +def _seg_21(): + return [ (0x205F, '3', u' '), (0x2060, 'I'), (0x2061, 'X'), @@ -2204,6 +2290,10 @@ uts46data = ( (0x2133, 'M', u'm'), (0x2134, 'M', u'o'), (0x2135, 'M', u'א'), + ] + +def _seg_22(): + return [ (0x2136, 'M', u'ב'), (0x2137, 'M', u'ג'), (0x2138, 'M', u'ד'), @@ -2304,6 +2394,10 @@ uts46data = ( (0x2469, 'M', u'10'), (0x246A, 'M', u'11'), (0x246B, 'M', u'12'), + ] + +def _seg_23(): + return [ (0x246C, 'M', u'13'), (0x246D, 'M', u'14'), (0x246E, 'M', u'15'), @@ -2404,6 +2498,10 @@ uts46data = ( (0x24E0, 'M', u'q'), (0x24E1, 'M', u'r'), (0x24E2, 'M', u's'), + ] + +def _seg_24(): + return [ (0x24E3, 'M', u't'), (0x24E4, 'M', u'u'), (0x24E5, 'M', u'v'), @@ -2504,6 +2602,10 @@ uts46data = ( (0x2C80, 'M', u'ⲁ'), (0x2C81, 'V'), (0x2C82, 'M', u'ⲃ'), + ] + +def _seg_25(): + return [ (0x2C83, 'V'), (0x2C84, 'M', u'ⲅ'), (0x2C85, 'V'), @@ -2604,6 +2706,10 @@ uts46data = ( (0x2CEB, 'M', u'ⳬ'), (0x2CEC, 'V'), (0x2CED, 'M', u'ⳮ'), + ] + +def _seg_26(): + return [ (0x2CEE, 'V'), (0x2CF2, 'M', u'ⳳ'), (0x2CF3, 'V'), @@ -2704,6 +2810,10 @@ uts46data = ( (0x2F37, 'M', u'弋'), (0x2F38, 'M', u'弓'), (0x2F39, 'M', u'彐'), + ] + +def _seg_27(): + return [ (0x2F3A, 'M', u'彡'), (0x2F3B, 'M', u'彳'), (0x2F3C, 'M', u'心'), @@ -2804,6 +2914,10 @@ uts46data = ( (0x2F9B, 'M', u'走'), (0x2F9C, 'M', u'足'), (0x2F9D, 'M', u'身'), + ] + +def _seg_28(): + return [ (0x2F9E, 'M', u'車'), (0x2F9F, 'M', u'辛'), (0x2FA0, 'M', u'辰'), @@ -2904,6 +3018,10 @@ uts46data = ( (0x3142, 'M', u'ᄇ'), (0x3143, 'M', u'ᄈ'), (0x3144, 'M', u'ᄡ'), + ] + +def _seg_29(): + return [ (0x3145, 'M', u'ᄉ'), (0x3146, 'M', u'ᄊ'), (0x3147, 'M', u'ᄋ'), @@ -3004,6 +3122,10 @@ uts46data = ( (0x3202, '3', u'(ᄃ)'), (0x3203, '3', u'(ᄅ)'), (0x3204, '3', u'(ᄆ)'), + ] + +def _seg_30(): + return [ (0x3205, '3', u'(ᄇ)'), (0x3206, '3', u'(ᄉ)'), (0x3207, '3', u'(ᄋ)'), @@ -3104,6 +3226,10 @@ uts46data = ( (0x326D, 'M', u'ᄒ'), (0x326E, 'M', u'가'), (0x326F, 'M', u'나'), + ] + +def _seg_31(): + return [ (0x3270, 'M', u'다'), (0x3271, 'M', u'라'), (0x3272, 'M', u'마'), @@ -3204,6 +3330,10 @@ uts46data = ( (0x32D1, 'M', u'イ'), (0x32D2, 'M', u'ウ'), (0x32D3, 'M', u'エ'), + ] + +def _seg_32(): + return [ (0x32D4, 'M', u'オ'), (0x32D5, 'M', u'カ'), (0x32D6, 'M', u'キ'), @@ -3304,6 +3434,10 @@ uts46data = ( (0x3335, 'M', u'フラン'), (0x3336, 'M', u'ヘクタール'), (0x3337, 'M', u'ペソ'), + ] + +def _seg_33(): + return [ (0x3338, 'M', u'ペニヒ'), (0x3339, 'M', u'ヘルツ'), (0x333A, 'M', u'ペンス'), @@ -3404,6 +3538,10 @@ uts46data = ( (0x3399, 'M', u'fm'), (0x339A, 'M', u'nm'), (0x339B, 'M', u'μm'), + ] + +def _seg_34(): + return [ (0x339C, 'M', u'mm'), (0x339D, 'M', u'cm'), (0x339E, 'M', u'km'), @@ -3504,6 +3642,10 @@ uts46data = ( (0x33FD, 'M', u'30日'), (0x33FE, 'M', u'31日'), (0x33FF, 'M', u'gal'), + ] + +def _seg_35(): + return [ (0x3400, 'V'), (0x4DB6, 'X'), (0x4DC0, 'V'), @@ -3604,6 +3746,10 @@ uts46data = ( (0xA72F, 'V'), (0xA732, 'M', u'ꜳ'), (0xA733, 'V'), + ] + +def _seg_36(): + return [ (0xA734, 'M', u'ꜵ'), (0xA735, 'V'), (0xA736, 'M', u'ꜷ'), @@ -3704,6 +3850,10 @@ uts46data = ( (0xA7AA, 'M', u'ɦ'), (0xA7AB, 'X'), (0xA7F8, 'M', u'ħ'), + ] + +def _seg_37(): + return [ (0xA7F9, 'M', u'œ'), (0xA7FA, 'V'), (0xA82C, 'X'), @@ -3804,6 +3954,10 @@ uts46data = ( (0xF92B, 'M', u'狼'), (0xF92C, 'M', u'郎'), (0xF92D, 'M', u'來'), + ] + +def _seg_38(): + return [ (0xF92E, 'M', u'冷'), (0xF92F, 'M', u'勞'), (0xF930, 'M', u'擄'), @@ -3904,6 +4058,10 @@ uts46data = ( (0xF98F, 'M', u'憐'), (0xF990, 'M', u'戀'), (0xF991, 'M', u'撚'), + ] + +def _seg_39(): + return [ (0xF992, 'M', u'漣'), (0xF993, 'M', u'煉'), (0xF994, 'M', u'璉'), @@ -4004,6 +4162,10 @@ uts46data = ( (0xF9F3, 'M', u'麟'), (0xF9F4, 'M', u'林'), (0xF9F5, 'M', u'淋'), + ] + +def _seg_40(): + return [ (0xF9F6, 'M', u'臨'), (0xF9F7, 'M', u'立'), (0xF9F8, 'M', u'笠'), @@ -4104,6 +4266,10 @@ uts46data = ( (0xFA5C, 'M', u'臭'), (0xFA5D, 'M', u'艹'), (0xFA5F, 'M', u'著'), + ] + +def _seg_41(): + return [ (0xFA60, 'M', u'褐'), (0xFA61, 'M', u'視'), (0xFA62, 'M', u'謁'), @@ -4204,6 +4370,10 @@ uts46data = ( (0xFAC2, 'M', u'輸'), (0xFAC3, 'M', u'遲'), (0xFAC4, 'M', u'醙'), + ] + +def _seg_42(): + return [ (0xFAC5, 'M', u'鉶'), (0xFAC6, 'M', u'陼'), (0xFAC7, 'M', u'難'), @@ -4304,6 +4474,10 @@ uts46data = ( (0xFB7A, 'M', u'چ'), (0xFB7E, 'M', u'ڇ'), (0xFB82, 'M', u'ڍ'), + ] + +def _seg_43(): + return [ (0xFB84, 'M', u'ڌ'), (0xFB86, 'M', u'ڎ'), (0xFB88, 'M', u'ڈ'), @@ -4404,6 +4578,10 @@ uts46data = ( (0xFC3C, 'M', u'كم'), (0xFC3D, 'M', u'كى'), (0xFC3E, 'M', u'كي'), + ] + +def _seg_44(): + return [ (0xFC3F, 'M', u'لج'), (0xFC40, 'M', u'لح'), (0xFC41, 'M', u'لخ'), @@ -4504,6 +4682,10 @@ uts46data = ( (0xFCA0, 'M', u'به'), (0xFCA1, 'M', u'تج'), (0xFCA2, 'M', u'تح'), + ] + +def _seg_45(): + return [ (0xFCA3, 'M', u'تخ'), (0xFCA4, 'M', u'تم'), (0xFCA5, 'M', u'ته'), @@ -4604,6 +4786,10 @@ uts46data = ( (0xFD04, 'M', u'خي'), (0xFD05, 'M', u'صى'), (0xFD06, 'M', u'صي'), + ] + +def _seg_46(): + return [ (0xFD07, 'M', u'ضى'), (0xFD08, 'M', u'ضي'), (0xFD09, 'M', u'شج'), @@ -4704,6 +4890,10 @@ uts46data = ( (0xFD87, 'M', u'لمح'), (0xFD89, 'M', u'محج'), (0xFD8A, 'M', u'محم'), + ] + +def _seg_47(): + return [ (0xFD8B, 'M', u'محي'), (0xFD8C, 'M', u'مجح'), (0xFD8D, 'M', u'مجم'), @@ -4804,6 +4994,10 @@ uts46data = ( (0xFE3C, 'M', u'】'), (0xFE3D, 'M', u'《'), (0xFE3E, 'M', u'》'), + ] + +def _seg_48(): + return [ (0xFE3F, 'M', u'〈'), (0xFE40, 'M', u'〉'), (0xFE41, 'M', u'「'), @@ -4904,6 +5098,10 @@ uts46data = ( (0xFF00, 'X'), (0xFF01, '3', u'!'), (0xFF02, '3', u'"'), + ] + +def _seg_49(): + return [ (0xFF03, '3', u'#'), (0xFF04, '3', u'$'), (0xFF05, '3', u'%'), @@ -5004,6 +5202,10 @@ uts46data = ( (0xFF64, 'M', u'、'), (0xFF65, 'M', u'・'), (0xFF66, 'M', u'ヲ'), + ] + +def _seg_50(): + return [ (0xFF67, 'M', u'ァ'), (0xFF68, 'M', u'ィ'), (0xFF69, 'M', u'ゥ'), @@ -5104,6 +5306,10 @@ uts46data = ( (0xFFCB, 'M', u'ᅨ'), (0xFFCC, 'M', u'ᅩ'), (0xFFCD, 'M', u'ᅪ'), + ] + +def _seg_51(): + return [ (0xFFCE, 'M', u'ᅫ'), (0xFFCF, 'M', u'ᅬ'), (0xFFD0, 'X'), @@ -5204,6 +5410,10 @@ uts46data = ( (0x1041B, 'M', u'𐑃'), (0x1041C, 'M', u'𐑄'), (0x1041D, 'M', u'𐑅'), + ] + +def _seg_52(): + return [ (0x1041E, 'M', u'𐑆'), (0x1041F, 'M', u'𐑇'), (0x10420, 'M', u'𐑈'), @@ -5304,6 +5514,10 @@ uts46data = ( (0x12474, 'X'), (0x13000, 'V'), (0x1342F, 'X'), + ] + +def _seg_53(): + return [ (0x16800, 'V'), (0x16A39, 'X'), (0x16F00, 'V'), @@ -5404,6 +5618,10 @@ uts46data = ( (0x1D43A, 'M', u'g'), (0x1D43B, 'M', u'h'), (0x1D43C, 'M', u'i'), + ] + +def _seg_54(): + return [ (0x1D43D, 'M', u'j'), (0x1D43E, 'M', u'k'), (0x1D43F, 'M', u'l'), @@ -5504,6 +5722,10 @@ uts46data = ( (0x1D49E, 'M', u'c'), (0x1D49F, 'M', u'd'), (0x1D4A0, 'X'), + ] + +def _seg_55(): + return [ (0x1D4A2, 'M', u'g'), (0x1D4A3, 'X'), (0x1D4A5, 'M', u'j'), @@ -5604,6 +5826,10 @@ uts46data = ( (0x1D505, 'M', u'b'), (0x1D506, 'X'), (0x1D507, 'M', u'd'), + ] + +def _seg_56(): + return [ (0x1D508, 'M', u'e'), (0x1D509, 'M', u'f'), (0x1D50A, 'M', u'g'), @@ -5704,6 +5930,10 @@ uts46data = ( (0x1D56C, 'M', u'a'), (0x1D56D, 'M', u'b'), (0x1D56E, 'M', u'c'), + ] + +def _seg_57(): + return [ (0x1D56F, 'M', u'd'), (0x1D570, 'M', u'e'), (0x1D571, 'M', u'f'), @@ -5804,6 +6034,10 @@ uts46data = ( (0x1D5D0, 'M', u'w'), (0x1D5D1, 'M', u'x'), (0x1D5D2, 'M', u'y'), + ] + +def _seg_58(): + return [ (0x1D5D3, 'M', u'z'), (0x1D5D4, 'M', u'a'), (0x1D5D5, 'M', u'b'), @@ -5904,6 +6138,10 @@ uts46data = ( (0x1D634, 'M', u's'), (0x1D635, 'M', u't'), (0x1D636, 'M', u'u'), + ] + +def _seg_59(): + return [ (0x1D637, 'M', u'v'), (0x1D638, 'M', u'w'), (0x1D639, 'M', u'x'), @@ -6004,6 +6242,10 @@ uts46data = ( (0x1D698, 'M', u'o'), (0x1D699, 'M', u'p'), (0x1D69A, 'M', u'q'), + ] + +def _seg_60(): + return [ (0x1D69B, 'M', u'r'), (0x1D69C, 'M', u's'), (0x1D69D, 'M', u't'), @@ -6104,6 +6346,10 @@ uts46data = ( (0x1D6FE, 'M', u'γ'), (0x1D6FF, 'M', u'δ'), (0x1D700, 'M', u'ε'), + ] + +def _seg_61(): + return [ (0x1D701, 'M', u'ζ'), (0x1D702, 'M', u'η'), (0x1D703, 'M', u'θ'), @@ -6204,6 +6450,10 @@ uts46data = ( (0x1D764, 'M', u'ο'), (0x1D765, 'M', u'π'), (0x1D766, 'M', u'ρ'), + ] + +def _seg_62(): + return [ (0x1D767, 'M', u'θ'), (0x1D768, 'M', u'σ'), (0x1D769, 'M', u'τ'), @@ -6304,6 +6554,10 @@ uts46data = ( (0x1D7CA, 'M', u'ϝ'), (0x1D7CC, 'X'), (0x1D7CE, 'M', u'0'), + ] + +def _seg_63(): + return [ (0x1D7CF, 'M', u'1'), (0x1D7D0, 'M', u'2'), (0x1D7D1, 'M', u'3'), @@ -6404,6 +6658,10 @@ uts46data = ( (0x1EE30, 'M', u'ف'), (0x1EE31, 'M', u'ص'), (0x1EE32, 'M', u'ق'), + ] + +def _seg_64(): + return [ (0x1EE33, 'X'), (0x1EE34, 'M', u'ش'), (0x1EE35, 'M', u'ت'), @@ -6504,6 +6762,10 @@ uts46data = ( (0x1EEA2, 'M', u'ج'), (0x1EEA3, 'M', u'د'), (0x1EEA4, 'X'), + ] + +def _seg_65(): + return [ (0x1EEA5, 'M', u'و'), (0x1EEA6, 'M', u'ز'), (0x1EEA7, 'M', u'ح'), @@ -6604,6 +6866,10 @@ uts46data = ( (0x1F140, 'M', u'q'), (0x1F141, 'M', u'r'), (0x1F142, 'M', u's'), + ] + +def _seg_66(): + return [ (0x1F143, 'M', u't'), (0x1F144, 'M', u'u'), (0x1F145, 'M', u'v'), @@ -6704,6 +6970,10 @@ uts46data = ( (0x1F400, 'V'), (0x1F43F, 'X'), (0x1F440, 'V'), + ] + +def _seg_67(): + return [ (0x1F441, 'X'), (0x1F442, 'V'), (0x1F4F8, 'X'), @@ -6804,6 +7074,10 @@ uts46data = ( (0x2F84B, 'M', u'圖'), (0x2F84C, 'M', u'嘆'), (0x2F84D, 'M', u'圗'), + ] + +def _seg_68(): + return [ (0x2F84E, 'M', u'噑'), (0x2F84F, 'M', u'噴'), (0x2F850, 'M', u'切'), @@ -6904,6 +7178,10 @@ uts46data = ( (0x2F8B2, 'M', u'成'), (0x2F8B3, 'M', u'戛'), (0x2F8B4, 'M', u'扝'), + ] + +def _seg_69(): + return [ (0x2F8B5, 'M', u'抱'), (0x2F8B6, 'M', u'拔'), (0x2F8B7, 'M', u'捐'), @@ -7004,6 +7282,10 @@ uts46data = ( (0x2F916, 'M', u'㶖'), (0x2F917, 'M', u'灊'), (0x2F918, 'M', u'災'), + ] + +def _seg_70(): + return [ (0x2F919, 'M', u'灷'), (0x2F91A, 'M', u'炭'), (0x2F91B, 'M', u'𠔥'), @@ -7104,6 +7386,10 @@ uts46data = ( (0x2F97D, 'M', u'聠'), (0x2F97E, 'M', u'𦖨'), (0x2F97F, 'M', u'聰'), + ] + +def _seg_71(): + return [ (0x2F980, 'M', u'𣍟'), (0x2F981, 'M', u'䏕'), (0x2F982, 'M', u'育'), @@ -7204,6 +7490,10 @@ uts46data = ( (0x2F9E1, 'M', u'𨗭'), (0x2F9E2, 'M', u'邔'), (0x2F9E3, 'M', u'郱'), + ] + +def _seg_72(): + return [ (0x2F9E4, 'M', u'鄑'), (0x2F9E5, 'M', u'𨜮'), (0x2F9E6, 'M', u'鄛'), @@ -7264,4 +7554,80 @@ uts46data = ( (0x2FA1E, 'X'), (0xE0100, 'I'), (0xE01F0, 'X'), + ] + +uts46data = tuple( + _seg_0() + + _seg_1() + + _seg_2() + + _seg_3() + + _seg_4() + + _seg_5() + + _seg_6() + + _seg_7() + + _seg_8() + + _seg_9() + + _seg_10() + + _seg_11() + + _seg_12() + + _seg_13() + + _seg_14() + + _seg_15() + + _seg_16() + + _seg_17() + + _seg_18() + + _seg_19() + + _seg_20() + + _seg_21() + + _seg_22() + + _seg_23() + + _seg_24() + + _seg_25() + + _seg_26() + + _seg_27() + + _seg_28() + + _seg_29() + + _seg_30() + + _seg_31() + + _seg_32() + + _seg_33() + + _seg_34() + + _seg_35() + + _seg_36() + + _seg_37() + + _seg_38() + + _seg_39() + + _seg_40() + + _seg_41() + + _seg_42() + + _seg_43() + + _seg_44() + + _seg_45() + + _seg_46() + + _seg_47() + + _seg_48() + + _seg_49() + + _seg_50() + + _seg_51() + + _seg_52() + + _seg_53() + + _seg_54() + + _seg_55() + + _seg_56() + + _seg_57() + + _seg_58() + + _seg_59() + + _seg_60() + + _seg_61() + + _seg_62() + + _seg_63() + + _seg_64() + + _seg_65() + + _seg_66() + + _seg_67() + + _seg_68() + + _seg_69() + + _seg_70() + + _seg_71() + + _seg_72() ) diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index 6afd39b4..ac3ff838 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -32,7 +32,7 @@ except ImportError: __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.19.1' +__version__ = '1.20' __all__ = ( 'HTTPConnectionPool', diff --git a/requests/packages/urllib3/connection.py b/requests/packages/urllib3/connection.py index e24f5e32..9f06c395 100644 --- a/requests/packages/urllib3/connection.py +++ b/requests/packages/urllib3/connection.py @@ -56,7 +56,10 @@ port_by_scheme = { 'https': 443, } -RECENT_DATE = datetime.date(2014, 1, 1) +# When updating RECENT_DATE, move it to +# within two years of the current date, and no +# earlier than 6 months ago. +RECENT_DATE = datetime.date(2016, 1, 1) class DummyConnection(object): diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 19d08f23..b4f1166a 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -25,7 +25,7 @@ from .exceptions import ( ) from .packages.ssl_match_hostname import CertificateError from .packages import six -from .packages.six.moves.queue import LifoQueue, Empty, Full +from .packages.six.moves import queue from .connection import ( port_by_scheme, DummyConnection, @@ -36,6 +36,7 @@ from .request import RequestMethods from .response import HTTPResponse from .util.connection import is_connection_dropped +from .util.request import set_file_position from .util.response import assert_header_parsing from .util.retry import Retry from .util.timeout import Timeout @@ -61,19 +62,13 @@ class ConnectionPool(object): """ scheme = None - QueueCls = LifoQueue + QueueCls = queue.LifoQueue def __init__(self, host, port=None): if not host: raise LocationValueError("No host specified.") - # httplib doesn't like it when we include brackets in ipv6 addresses - # Specifically, if we include brackets but also pass the port then - # httplib crazily doubles up the square brackets on the Host header. - # Instead, we need to make sure we never pass ``None`` as the port. - # However, for backward compatibility reasons we can't actually - # *assert* that. - self.host = host.strip('[]') + self.host = _ipv6_host(host).lower() self.port = port def __str__(self): @@ -154,7 +149,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): A dictionary with proxy headers, should not be used directly, instead, see :class:`urllib3.connectionpool.ProxyManager`" - :param \**conn_kw: + :param \\**conn_kw: Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`, :class:`urllib3.connection.HTTPSConnection` instances. """ @@ -235,7 +230,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): except AttributeError: # self.pool is None raise ClosedPoolError(self, "Pool is closed.") - except Empty: + except queue.Empty: if self.block: raise EmptyPoolError(self, "Pool reached maximum size and no more " @@ -274,7 +269,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): except AttributeError: # self.pool is None. pass - except Full: + except queue.Full: # This should never happen if self.block == True log.warning( "Connection pool is full, discarding connection: %s", @@ -424,7 +419,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if conn: conn.close() - except Empty: + except queue.Empty: pass # Done. def is_same_host(self, url): @@ -438,6 +433,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # TODO: Add optional support for socket.gethostbyname checking. scheme, host, port = get_host(url) + host = _ipv6_host(host).lower() + # Use explicit default port for comparison when none is given if self.port and not port: port = port_by_scheme.get(scheme) @@ -449,7 +446,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): def urlopen(self, method, url, body=None, headers=None, retries=None, redirect=True, assert_same_host=True, timeout=_Default, pool_timeout=None, release_conn=None, chunked=False, - **response_kw): + body_pos=None, **response_kw): """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all @@ -531,7 +528,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. - :param \**response_kw: + :param int body_pos: + Position to seek to in file-like body in the event of a retry or + redirect. Typically this won't need to be set because urllib3 will + auto-populate the value when needed. + + :param \\**response_kw: Additional parameters are passed to :meth:`urllib3.response.HTTPResponse.from_httplib` """ @@ -576,6 +578,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # ensures we do proper cleanup in finally. clean_exit = False + # Rewind body position, if needed. Record current position + # for future rewinds in the event of a redirect/retry. + body_pos = set_file_position(body, body_pos) + try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) @@ -612,7 +618,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Everything went great! clean_exit = True - except Empty: + except queue.Empty: # Timed out by queue. raise EmptyPoolError(self, "No pool connections are available.") @@ -668,7 +674,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): return self.urlopen(method, url, body, headers, retries, redirect, assert_same_host, timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, **response_kw) + release_conn=release_conn, body_pos=body_pos, + **response_kw) # Handle redirect? redirect_location = redirect and response.get_redirect_location() @@ -693,7 +700,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): retries=retries, redirect=redirect, assert_same_host=assert_same_host, timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, **response_kw) + release_conn=release_conn, body_pos=body_pos, + **response_kw) # Check if we should retry the HTTP response. has_retry_after = bool(response.getheader('Retry-After')) @@ -714,7 +722,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): retries=retries, redirect=redirect, assert_same_host=assert_same_host, timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, **response_kw) + release_conn=release_conn, + body_pos=body_pos, **response_kw) return response @@ -853,7 +862,7 @@ def connection_from_url(url, **kw): :param url: Absolute URL string that must include the scheme. Port is optional. - :param \**kw: + :param \\**kw: Passes additional parameters to the constructor of the appropriate :class:`.ConnectionPool`. Useful for specifying things like timeout, maxsize, headers, etc. @@ -869,3 +878,22 @@ def connection_from_url(url, **kw): return HTTPSConnectionPool(host, port=port, **kw) else: return HTTPConnectionPool(host, port=port, **kw) + + +def _ipv6_host(host): + """ + Process IPv6 address literals + """ + + # httplib doesn't like it when we include brackets in IPv6 addresses + # Specifically, if we include brackets but also pass the port then + # httplib crazily doubles up the square brackets on the Host header. + # Instead, we need to make sure we never pass ``None`` as the port. + # However, for backward compatibility reasons we can't actually + # *assert* that. See http://bugs.python.org/issue28539 + # + # Also if an IPv6 address literal has a zone identifier, the + # percent sign might be URIencoded, convert it back into ASCII + if host.startswith('[') and host.endswith(']'): + host = host.replace('%25', '%').strip('[]') + return host diff --git a/requests/packages/urllib3/contrib/appengine.py b/requests/packages/urllib3/contrib/appengine.py index c3249eeb..814b0222 100644 --- a/requests/packages/urllib3/contrib/appengine.py +++ b/requests/packages/urllib3/contrib/appengine.py @@ -111,7 +111,7 @@ class AppEngineManager(RequestMethods): warnings.warn( "urllib3 is using URLFetch on Google App Engine sandbox instead " "of sockets. To use sockets directly instead of URLFetch see " - "https://urllib3.readthedocs.io/en/latest/contrib.html.", + "https://urllib3.readthedocs.io/en/latest/reference/urllib3.contrib.html.", AppEnginePlatformWarning) RequestMethods.__init__(self, headers) diff --git a/requests/packages/urllib3/contrib/pyopenssl.py b/requests/packages/urllib3/contrib/pyopenssl.py index c6db25d4..eb4d4765 100644 --- a/requests/packages/urllib3/contrib/pyopenssl.py +++ b/requests/packages/urllib3/contrib/pyopenssl.py @@ -43,7 +43,6 @@ set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable. """ from __future__ import absolute_import -import idna import OpenSSL.SSL from cryptography import x509 from cryptography.hazmat.backends.openssl import backend as openssl_backend @@ -60,7 +59,6 @@ except ImportError: # Platform-specific: Python 3 import logging import ssl -import select import six import sys @@ -111,6 +109,8 @@ log = logging.getLogger(__name__) def inject_into_urllib3(): 'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.' + _validate_dependencies_met() + util.ssl_.SSLContext = PyOpenSSLContext util.HAS_SNI = HAS_SNI util.ssl_.HAS_SNI = HAS_SNI @@ -128,6 +128,26 @@ def extract_from_urllib3(): util.ssl_.IS_PYOPENSSL = False +def _validate_dependencies_met(): + """ + Verifies that PyOpenSSL's package-level dependencies have been met. + Throws `ImportError` if they are not met. + """ + # Method added in `cryptography==1.1`; not available in older versions + from cryptography.x509.extensions import Extensions + if getattr(Extensions, "get_extension_for_class", None) is None: + raise ImportError("'cryptography' module missing required functionality. " + "Try upgrading to v1.3.4 or newer.") + + # pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509 + # attribute is only present on those versions. + from OpenSSL.crypto import X509 + x509 = X509() + if getattr(x509, "_x509", None) is None: + raise ImportError("'pyOpenSSL' module missing required functionality. " + "Try upgrading to v0.14 or newer.") + + def _dnsname_to_stdlib(name): """ Converts a dNSName SubjectAlternativeName field to the form used by the @@ -144,6 +164,8 @@ def _dnsname_to_stdlib(name): that we can't just safely call `idna.encode`: it can explode for wildcard names. This avoids that problem. """ + import idna + for prefix in [u'*.', u'.']: if name.startswith(prefix): name = name[len(prefix):] @@ -242,8 +264,7 @@ class WrappedSocket(object): else: raise except OpenSSL.SSL.WantReadError: - rd, wd, ed = select.select( - [self.socket], [], [], self.socket.gettimeout()) + rd = util.wait_for_read(self.socket, self.socket.gettimeout()) if not rd: raise timeout('The read operation timed out') else: @@ -265,8 +286,7 @@ class WrappedSocket(object): else: raise except OpenSSL.SSL.WantReadError: - rd, wd, ed = select.select( - [self.socket], [], [], self.socket.gettimeout()) + rd = util.wait_for_read(self.socket, self.socket.gettimeout()) if not rd: raise timeout('The read operation timed out') else: @@ -280,9 +300,8 @@ class WrappedSocket(object): try: return self.connection.send(data) except OpenSSL.SSL.WantWriteError: - _, wlist, _ = select.select([], [self.socket], [], - self.socket.gettimeout()) - if not wlist: + wr = util.wait_for_write(self.socket, self.socket.gettimeout()) + if not wr: raise timeout() continue @@ -416,7 +435,7 @@ class PyOpenSSLContext(object): try: cnx.do_handshake() except OpenSSL.SSL.WantReadError: - rd, _, _ = select.select([sock], [], [], sock.gettimeout()) + rd = util.wait_for_read(sock, sock.gettimeout()) if not rd: raise timeout('select timed out') continue diff --git a/requests/packages/urllib3/contrib/socks.py b/requests/packages/urllib3/contrib/socks.py index c8fa8409..39e92fde 100644 --- a/requests/packages/urllib3/contrib/socks.py +++ b/requests/packages/urllib3/contrib/socks.py @@ -83,6 +83,7 @@ class SOCKSConnection(HTTPConnection): proxy_port=self._socks_options['proxy_port'], proxy_username=self._socks_options['username'], proxy_password=self._socks_options['password'], + proxy_rdns=self._socks_options['rdns'], timeout=self.timeout, **extra_kw ) @@ -153,8 +154,16 @@ class SOCKSProxyManager(PoolManager): if parsed.scheme == 'socks5': socks_version = socks.PROXY_TYPE_SOCKS5 + rdns = False + elif parsed.scheme == 'socks5h': + socks_version = socks.PROXY_TYPE_SOCKS5 + rdns = True elif parsed.scheme == 'socks4': socks_version = socks.PROXY_TYPE_SOCKS4 + rdns = False + elif parsed.scheme == 'socks4a': + socks_version = socks.PROXY_TYPE_SOCKS4 + rdns = True else: raise ValueError( "Unable to determine SOCKS version from %s" % proxy_url @@ -168,6 +177,7 @@ class SOCKSProxyManager(PoolManager): 'proxy_port': parsed.port, 'username': username, 'password': password, + 'rdns': rdns } connection_pool_kw['_socks_options'] = socks_options diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py index 8a091c17..6c4be581 100644 --- a/requests/packages/urllib3/exceptions.py +++ b/requests/packages/urllib3/exceptions.py @@ -239,3 +239,8 @@ class HeaderParsingError(HTTPError): def __init__(self, defects, unparsed_data): message = '%s, unparsed data: %r' % (defects or 'Unknown', unparsed_data) super(HeaderParsingError, self).__init__(message) + + +class UnrewindableBodyError(HTTPError): + "urllib3 encountered an error when trying to rewind a body" + pass diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index 276b54dd..cc5a00e4 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -93,7 +93,7 @@ class PoolManager(RequestMethods): Headers to include with all requests, unless other headers are given explicitly. - :param \**connection_pool_kw: + :param \\**connection_pool_kw: Additional parameters are used to create fresh :class:`urllib3.connectionpool.ConnectionPool` instances. diff --git a/requests/packages/urllib3/util/__init__.py b/requests/packages/urllib3/util/__init__.py index 4778cf99..5ced5a44 100644 --- a/requests/packages/urllib3/util/__init__.py +++ b/requests/packages/urllib3/util/__init__.py @@ -24,6 +24,10 @@ from .url import ( split_first, Url, ) +from .wait import ( + wait_for_read, + wait_for_write +) __all__ = ( 'HAS_SNI', @@ -43,4 +47,6 @@ __all__ = ( 'resolve_ssl_version', 'split_first', 'ssl_wrap_socket', + 'wait_for_read', + 'wait_for_write' ) diff --git a/requests/packages/urllib3/util/connection.py b/requests/packages/urllib3/util/connection.py index 3d608772..bf699cfd 100644 --- a/requests/packages/urllib3/util/connection.py +++ b/requests/packages/urllib3/util/connection.py @@ -1,13 +1,7 @@ from __future__ import absolute_import import socket -try: - from select import poll, POLLIN -except ImportError: # `poll` doesn't exist on OSX and other platforms - poll = False - try: - from select import select - except ImportError: # `select` doesn't exist on AppEngine. - select = False +from .wait import wait_for_read +from .selectors import HAS_SELECT, SelectorError def is_connection_dropped(conn): # Platform-specific @@ -26,22 +20,13 @@ def is_connection_dropped(conn): # Platform-specific if sock is None: # Connection already closed (such as by httplib). return True - if not poll: - if not select: # Platform-specific: AppEngine - return False + if not HAS_SELECT: + return False - try: - return select([sock], [], [], 0.0)[0] - except socket.error: - return True - - # This version is better on platforms that support it. - p = poll() - p.register(sock, POLLIN) - for (fno, ev) in p.poll(0.0): - if fno == sock.fileno(): - # Either data is buffered (bad), or the connection is dropped. - return True + try: + return bool(wait_for_read(sock, timeout=0.0)) + except SelectorError: + return True # This function is copied from socket.py in the Python 2.7 standard diff --git a/requests/packages/urllib3/util/request.py b/requests/packages/urllib3/util/request.py index 73779315..974fc40a 100644 --- a/requests/packages/urllib3/util/request.py +++ b/requests/packages/urllib3/util/request.py @@ -1,9 +1,11 @@ from __future__ import absolute_import from base64 import b64encode -from ..packages.six import b +from ..packages.six import b, integer_types +from ..exceptions import UnrewindableBodyError ACCEPT_ENCODING = 'gzip,deflate' +_FAILEDTELL = object() def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, @@ -70,3 +72,47 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, headers['cache-control'] = 'no-cache' return headers + + +def set_file_position(body, pos): + """ + If a position is provided, move file to that point. + Otherwise, we'll attempt to record a position for future use. + """ + if pos is not None: + rewind_body(body, pos) + elif getattr(body, 'tell', None) is not None: + try: + pos = body.tell() + except (IOError, OSError): + # This differentiates from None, allowing us to catch + # a failed `tell()` later when trying to rewind the body. + pos = _FAILEDTELL + + return pos + + +def rewind_body(body, body_pos): + """ + Attempt to rewind body to a certain position. + Primarily used for request redirects and retries. + + :param body: + File-like object that supports seek. + + :param int pos: + Position to seek to in file. + """ + body_seek = getattr(body, 'seek', None) + if body_seek is not None and isinstance(body_pos, integer_types): + try: + body_seek(body_pos) + except (IOError, OSError): + raise UnrewindableBodyError("An error occured when rewinding request " + "body for redirect/retry.") + elif body_pos is _FAILEDTELL: + raise UnrewindableBodyError("Unable to record file position for rewinding " + "request body during a redirect/retry.") + else: + raise ValueError("body_pos must be of type integer, " + "instead it was %s." % type(body_pos)) diff --git a/requests/packages/urllib3/util/retry.py b/requests/packages/urllib3/util/retry.py index 47ad539a..c9e7d287 100644 --- a/requests/packages/urllib3/util/retry.py +++ b/requests/packages/urllib3/util/retry.py @@ -273,12 +273,25 @@ class Retry(object): """ return isinstance(err, (ReadTimeoutError, ProtocolError)) - def is_retry(self, method, status_code, has_retry_after=False): - """ Is this method/status code retryable? (Based on method/codes whitelists) + def _is_method_retryable(self, method): + """ Checks if a given HTTP method should be retried upon, depending if + it is included on the method whitelist. """ if self.method_whitelist and method.upper() not in self.method_whitelist: return False + return True + + def is_retry(self, method, status_code, has_retry_after=False): + """ Is this method/status code retryable? (Based on whitelists and control + variables such as the number of total retries to allow, whether to + respect the Retry-After header, whether this header is present, and + whether the returned status code is on the list of status codes to + be retried upon on the presence of the aforementioned header) + """ + if not self._is_method_retryable(method): + return False + if self.status_forcelist and status_code in self.status_forcelist: return True @@ -330,7 +343,7 @@ class Retry(object): elif error and self._is_read_error(error): # Read retry? - if read is False: + if read is False or not self._is_method_retryable(method): raise six.reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 diff --git a/requests/packages/urllib3/util/selectors.py b/requests/packages/urllib3/util/selectors.py new file mode 100644 index 00000000..51208b69 --- /dev/null +++ b/requests/packages/urllib3/util/selectors.py @@ -0,0 +1,524 @@ +# Backport of selectors.py from Python 3.5+ to support Python < 3.4 +# Also has the behavior specified in PEP 475 which is to retry syscalls +# in the case of an EINTR error. This module is required because selectors34 +# does not follow this behavior and instead returns that no dile descriptor +# events have occurred rather than retry the syscall. The decision to drop +# support for select.devpoll is made to maintain 100% test coverage. + +import errno +import math +import select +from collections import namedtuple, Mapping + +import time +try: + monotonic = time.monotonic +except (AttributeError, ImportError): # Python 3.3< + monotonic = time.time + +EVENT_READ = (1 << 0) +EVENT_WRITE = (1 << 1) + +HAS_SELECT = True # Variable that shows whether the platform has a selector. +_SYSCALL_SENTINEL = object() # Sentinel in case a system call returns None. + + +class SelectorError(Exception): + def __init__(self, errcode): + super(SelectorError, self).__init__() + self.errno = errcode + + def __repr__(self): + return "".format(self.errno) + + def __str__(self): + return self.__repr__() + + +def _fileobj_to_fd(fileobj): + """ Return a file descriptor from a file object. If + given an integer will simply return that integer back. """ + if isinstance(fileobj, int): + fd = fileobj + else: + try: + fd = int(fileobj.fileno()) + except (AttributeError, TypeError, ValueError): + raise ValueError("Invalid file object: {0!r}".format(fileobj)) + if fd < 0: + raise ValueError("Invalid file descriptor: {0}".format(fd)) + return fd + + +def _syscall_wrapper(func, recalc_timeout, *args, **kwargs): + """ Wrapper function for syscalls that could fail due to EINTR. + All functions should be retried if there is time left in the timeout + in accordance with PEP 475. """ + timeout = kwargs.get("timeout", None) + if timeout is None: + expires = None + recalc_timeout = False + else: + timeout = float(timeout) + if timeout < 0.0: # Timeout less than 0 treated as no timeout. + expires = None + else: + expires = monotonic() + timeout + + args = list(args) + if recalc_timeout and "timeout" not in kwargs: + raise ValueError( + "Timeout must be in args or kwargs to be recalculated") + + result = _SYSCALL_SENTINEL + while result is _SYSCALL_SENTINEL: + try: + result = func(*args, **kwargs) + # OSError is thrown by select.select + # IOError is thrown by select.epoll.poll + # select.error is thrown by select.poll.poll + # Aren't we thankful for Python 3.x rework for exceptions? + except (OSError, IOError, select.error) as e: + # select.error wasn't a subclass of OSError in the past. + errcode = None + if hasattr(e, "errno"): + errcode = e.errno + elif hasattr(e, "args"): + errcode = e.args[0] + + # Also test for the Windows equivalent of EINTR. + is_interrupt = (errcode == errno.EINTR or (hasattr(errno, "WSAEINTR") and + errcode == errno.WSAEINTR)) + + if is_interrupt: + if expires is not None: + current_time = monotonic() + if current_time > expires: + raise OSError(errno=errno.ETIMEDOUT) + if recalc_timeout: + if "timeout" in kwargs: + kwargs["timeout"] = expires - current_time + continue + if errcode: + raise SelectorError(errcode) + else: + raise + return result + + +SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data']) + + +class _SelectorMapping(Mapping): + """ Mapping of file objects to selector keys """ + + def __init__(self, selector): + self._selector = selector + + def __len__(self): + return len(self._selector._fd_to_key) + + def __getitem__(self, fileobj): + try: + fd = self._selector._fileobj_lookup(fileobj) + return self._selector._fd_to_key[fd] + except KeyError: + raise KeyError("{0!r} is not registered.".format(fileobj)) + + def __iter__(self): + return iter(self._selector._fd_to_key) + + +class BaseSelector(object): + """ Abstract Selector class + + A selector supports registering file objects to be monitored + for specific I/O events. + + A file object is a file descriptor or any object with a + `fileno()` method. An arbitrary object can be attached to the + file object which can be used for example to store context info, + a callback, etc. + + A selector can use various implementations (select(), poll(), epoll(), + and kqueue()) depending on the platform. The 'DefaultSelector' class uses + the most efficient implementation for the current platform. + """ + def __init__(self): + # Maps file descriptors to keys. + self._fd_to_key = {} + + # Read-only mapping returned by get_map() + self._map = _SelectorMapping(self) + + def _fileobj_lookup(self, fileobj): + """ Return a file descriptor from a file object. + This wraps _fileobj_to_fd() to do an exhaustive + search in case the object is invalid but we still + have it in our map. Used by unregister() so we can + unregister an object that was previously registered + even if it is closed. It is also used by _SelectorMapping + """ + try: + return _fileobj_to_fd(fileobj) + except ValueError: + + # Search through all our mapped keys. + for key in self._fd_to_key.values(): + if key.fileobj is fileobj: + return key.fd + + # Raise ValueError after all. + raise + + def register(self, fileobj, events, data=None): + """ Register a file object for a set of events to monitor. """ + if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)): + raise ValueError("Invalid events: {0!r}".format(events)) + + key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data) + + if key.fd in self._fd_to_key: + raise KeyError("{0!r} (FD {1}) is already registered" + .format(fileobj, key.fd)) + + self._fd_to_key[key.fd] = key + return key + + def unregister(self, fileobj): + """ Unregister a file object from being monitored. """ + try: + key = self._fd_to_key.pop(self._fileobj_lookup(fileobj)) + except KeyError: + raise KeyError("{0!r} is not registered".format(fileobj)) + return key + + def modify(self, fileobj, events, data=None): + """ Change a registered file object monitored events and data. """ + # NOTE: Some subclasses optimize this operation even further. + try: + key = self._fd_to_key[self._fileobj_lookup(fileobj)] + except KeyError: + raise KeyError("{0!r} is not registered".format(fileobj)) + + if events != key.events: + self.unregister(fileobj) + key = self.register(fileobj, events, data) + + elif data != key.data: + # Use a shortcut to update the data. + key = key._replace(data=data) + self._fd_to_key[key.fd] = key + + return key + + def select(self, timeout=None): + """ Perform the actual selection until some monitored file objects + are ready or the timeout expires. """ + raise NotImplementedError() + + def close(self): + """ Close the selector. This must be called to ensure that all + underlying resources are freed. """ + self._fd_to_key.clear() + self._map = None + + def get_key(self, fileobj): + """ Return the key associated with a registered file object. """ + mapping = self.get_map() + if mapping is None: + raise RuntimeError("Selector is closed") + try: + return mapping[fileobj] + except KeyError: + raise KeyError("{0!r} is not registered".format(fileobj)) + + def get_map(self): + """ Return a mapping of file objects to selector keys """ + return self._map + + def _key_from_fd(self, fd): + """ Return the key associated to a given file descriptor + Return None if it is not found. """ + try: + return self._fd_to_key[fd] + except KeyError: + return None + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +# Almost all platforms have select.select() +if hasattr(select, "select"): + class SelectSelector(BaseSelector): + """ Select-based selector. """ + def __init__(self): + super(SelectSelector, self).__init__() + self._readers = set() + self._writers = set() + + def register(self, fileobj, events, data=None): + key = super(SelectSelector, self).register(fileobj, events, data) + if events & EVENT_READ: + self._readers.add(key.fd) + if events & EVENT_WRITE: + self._writers.add(key.fd) + return key + + def unregister(self, fileobj): + key = super(SelectSelector, self).unregister(fileobj) + self._readers.discard(key.fd) + self._writers.discard(key.fd) + return key + + def _select(self, r, w, timeout=None): + """ Wrapper for select.select because timeout is a positional arg """ + return select.select(r, w, [], timeout) + + def select(self, timeout=None): + # Selecting on empty lists on Windows errors out. + if not len(self._readers) and not len(self._writers): + return [] + + timeout = None if timeout is None else max(timeout, 0.0) + ready = [] + r, w, _ = _syscall_wrapper(self._select, True, self._readers, + self._writers, timeout) + r = set(r) + w = set(w) + for fd in r | w: + events = 0 + if fd in r: + events |= EVENT_READ + if fd in w: + events |= EVENT_WRITE + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + +if hasattr(select, "poll"): + class PollSelector(BaseSelector): + """ Poll-based selector """ + def __init__(self): + super(PollSelector, self).__init__() + self._poll = select.poll() + + def register(self, fileobj, events, data=None): + key = super(PollSelector, self).register(fileobj, events, data) + event_mask = 0 + if events & EVENT_READ: + event_mask |= select.POLLIN + if events & EVENT_WRITE: + event_mask |= select.POLLOUT + self._poll.register(key.fd, event_mask) + return key + + def unregister(self, fileobj): + key = super(PollSelector, self).unregister(fileobj) + self._poll.unregister(key.fd) + return key + + def _wrap_poll(self, timeout=None): + """ Wrapper function for select.poll.poll() so that + _syscall_wrapper can work with only seconds. """ + if timeout is not None: + if timeout <= 0: + timeout = 0 + else: + # select.poll.poll() has a resolution of 1 millisecond, + # round away from zero to wait *at least* timeout seconds. + timeout = math.ceil(timeout * 1e3) + + result = self._poll.poll(timeout) + return result + + def select(self, timeout=None): + ready = [] + fd_events = _syscall_wrapper(self._wrap_poll, True, timeout=timeout) + for fd, event_mask in fd_events: + events = 0 + if event_mask & ~select.POLLIN: + events |= EVENT_WRITE + if event_mask & ~select.POLLOUT: + events |= EVENT_READ + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + + return ready + + +if hasattr(select, "epoll"): + class EpollSelector(BaseSelector): + """ Epoll-based selector """ + def __init__(self): + super(EpollSelector, self).__init__() + self._epoll = select.epoll() + + def fileno(self): + return self._epoll.fileno() + + def register(self, fileobj, events, data=None): + key = super(EpollSelector, self).register(fileobj, events, data) + events_mask = 0 + if events & EVENT_READ: + events_mask |= select.EPOLLIN + if events & EVENT_WRITE: + events_mask |= select.EPOLLOUT + _syscall_wrapper(self._epoll.register, False, key.fd, events_mask) + return key + + def unregister(self, fileobj): + key = super(EpollSelector, self).unregister(fileobj) + try: + _syscall_wrapper(self._epoll.unregister, False, key.fd) + except SelectorError: + # This can occur when the fd was closed since registry. + pass + return key + + def select(self, timeout=None): + if timeout is not None: + if timeout <= 0: + timeout = 0.0 + else: + # select.epoll.poll() has a resolution of 1 millisecond + # but luckily takes seconds so we don't need a wrapper + # like PollSelector. Just for better rounding. + timeout = math.ceil(timeout * 1e3) * 1e-3 + timeout = float(timeout) + else: + timeout = -1.0 # epoll.poll() must have a float. + + # We always want at least 1 to ensure that select can be called + # with no file descriptors registered. Otherwise will fail. + max_events = max(len(self._fd_to_key), 1) + + ready = [] + fd_events = _syscall_wrapper(self._epoll.poll, True, + timeout=timeout, + maxevents=max_events) + for fd, event_mask in fd_events: + events = 0 + if event_mask & ~select.EPOLLIN: + events |= EVENT_WRITE + if event_mask & ~select.EPOLLOUT: + events |= EVENT_READ + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + def close(self): + self._epoll.close() + super(EpollSelector, self).close() + + +if hasattr(select, "kqueue"): + class KqueueSelector(BaseSelector): + """ Kqueue / Kevent-based selector """ + def __init__(self): + super(KqueueSelector, self).__init__() + self._kqueue = select.kqueue() + + def fileno(self): + return self._kqueue.fileno() + + def register(self, fileobj, events, data=None): + key = super(KqueueSelector, self).register(fileobj, events, data) + if events & EVENT_READ: + kevent = select.kevent(key.fd, + select.KQ_FILTER_READ, + select.KQ_EV_ADD) + + _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) + + if events & EVENT_WRITE: + kevent = select.kevent(key.fd, + select.KQ_FILTER_WRITE, + select.KQ_EV_ADD) + + _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) + + return key + + def unregister(self, fileobj): + key = super(KqueueSelector, self).unregister(fileobj) + if key.events & EVENT_READ: + kevent = select.kevent(key.fd, + select.KQ_FILTER_READ, + select.KQ_EV_DELETE) + try: + _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) + except SelectorError: + pass + if key.events & EVENT_WRITE: + kevent = select.kevent(key.fd, + select.KQ_FILTER_WRITE, + select.KQ_EV_DELETE) + try: + _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) + except SelectorError: + pass + + return key + + def select(self, timeout=None): + if timeout is not None: + timeout = max(timeout, 0) + + max_events = len(self._fd_to_key) * 2 + ready_fds = {} + + kevent_list = _syscall_wrapper(self._kqueue.control, True, + None, max_events, timeout) + + for kevent in kevent_list: + fd = kevent.ident + event_mask = kevent.filter + events = 0 + if event_mask == select.KQ_FILTER_READ: + events |= EVENT_READ + if event_mask == select.KQ_FILTER_WRITE: + events |= EVENT_WRITE + + key = self._key_from_fd(fd) + if key: + if key.fd not in ready_fds: + ready_fds[key.fd] = (key, events & key.events) + else: + old_events = ready_fds[key.fd][1] + ready_fds[key.fd] = (key, (events | old_events) & key.events) + + return list(ready_fds.values()) + + def close(self): + self._kqueue.close() + super(KqueueSelector, self).close() + + +# Choose the best implementation, roughly: +# kqueue == epoll > poll > select. Devpoll not supported. (See above) +# select() also can't accept a FD > FD_SETSIZE (usually around 1024) +if 'KqueueSelector' in globals(): # Platform-specific: Mac OS and BSD + DefaultSelector = KqueueSelector +elif 'EpollSelector' in globals(): # Platform-specific: Linux + DefaultSelector = EpollSelector +elif 'PollSelector' in globals(): # Platform-specific: Linux + DefaultSelector = PollSelector +elif 'SelectSelector' in globals(): # Platform-specific: Windows + DefaultSelector = SelectSelector +else: # Platform-specific: AppEngine + def no_selector(_): + raise ValueError("Platform does not have a selector") + DefaultSelector = no_selector + HAS_SELECT = False diff --git a/requests/packages/urllib3/util/timeout.py b/requests/packages/urllib3/util/timeout.py index 1eaa6a35..cec817e6 100644 --- a/requests/packages/urllib3/util/timeout.py +++ b/requests/packages/urllib3/util/timeout.py @@ -11,11 +11,8 @@ from ..exceptions import TimeoutStateError _Default = object() -def current_time(): - """ - Retrieve the current time. This function is mocked out in unit testing. - """ - return time.time() +# Use time.monotonic if available. +current_time = getattr(time, "monotonic", time.time) class Timeout(object): diff --git a/requests/packages/urllib3/util/wait.py b/requests/packages/urllib3/util/wait.py new file mode 100644 index 00000000..cb396e50 --- /dev/null +++ b/requests/packages/urllib3/util/wait.py @@ -0,0 +1,40 @@ +from .selectors import ( + HAS_SELECT, + DefaultSelector, + EVENT_READ, + EVENT_WRITE +) + + +def _wait_for_io_events(socks, events, timeout=None): + """ Waits for IO events to be available from a list of sockets + or optionally a single socket if passed in. Returns a list of + sockets that can be interacted with immediately. """ + if not HAS_SELECT: + raise ValueError('Platform does not have a selector') + if not isinstance(socks, list): + # Probably just a single socket. + if hasattr(socks, "fileno"): + socks = [socks] + # Otherwise it might be a non-list iterable. + else: + socks = list(socks) + with DefaultSelector() as selector: + for sock in socks: + selector.register(sock, events) + return [key[0].fileobj for key in + selector.select(timeout) if key[1] & events] + + +def wait_for_read(socks, timeout=None): + """ Waits for reading to be available from a list of sockets + or optionally a single socket if passed in. Returns a list of + sockets that can be read from immediately. """ + return _wait_for_io_events(socks, EVENT_READ, timeout) + + +def wait_for_write(socks, timeout=None): + """ Waits for writing to be available from a list of sockets + or optionally a single socket if passed in. Returns a list of + sockets that can be written to immediately. """ + return _wait_for_io_events(socks, EVENT_WRITE, timeout) diff --git a/requests/status_codes.py b/requests/status_codes.py index db2986bb..dee89190 100644 --- a/requests/status_codes.py +++ b/requests/status_codes.py @@ -87,5 +87,5 @@ codes = LookupDict(name='status_codes') for code, titles in _codes.items(): for title in titles: setattr(codes, title, code) - if not title.startswith('\\'): + if not title.startswith(('\\', '/')): setattr(codes, title.upper(), code) diff --git a/requests/utils.py b/requests/utils.py index 519a06ed..02fb22bd 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -739,7 +739,7 @@ def guess_json_utf(data): # easy as counting the nulls and from their location and count # determine the encoding. Also detect a BOM, if present. sample = data[:4] - if sample in (codecs.BOM_UTF32_LE, codecs.BOM32_BE): + if sample in (codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE): return 'utf-32' # BOM included if sample[:3] == codecs.BOM_UTF8: return 'utf-8-sig' # BOM included, MS style (discouraged) diff --git a/requirements-to-freeze.txt b/requirements-to-freeze.txt deleted file mode 100644 index e8b9e354..00000000 --- a/requirements-to-freeze.txt +++ /dev/null @@ -1,4 +0,0 @@ -pytest -pytest-cov -pytest-httpbin -sphinx \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 7f12747c..00000000 --- a/requirements.txt +++ /dev/null @@ -1,24 +0,0 @@ -alabaster==0.7.7 -Babel==2.2.0 -coverage==4.0.3 -decorator==4.0.9 -docutils==0.12 -Flask==0.10.1 -httpbin==0.5.0 -itsdangerous==0.24 -Jinja2==2.8 -MarkupSafe==0.23 -py==1.4.31 -Pygments==2.1.1 -PySocks==1.5.6 -pytest==2.8.7 -pytest-cov==2.2.1 -pytest-httpbin==0.2.0 -pytest-mock==0.11.0 -pytz==2015.7 -six==1.10.0 -snowballstemmer==1.2.1 -Sphinx==1.3.5 -sphinx-rtd-theme==0.1.9 -Werkzeug==0.11.4 -wheel==0.29.0 diff --git a/setup.py b/setup.py index c2406240..b1623489 100755 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ packages = [ ] requires = [] -test_requirements = ['pytest>=2.8.0', 'pytest-httpbin==0.0.7', 'pytest-cov'] +test_requirements = ['pytest>=2.8.0', 'pytest-httpbin==0.0.7', 'pytest-cov', 'pytest-mock'] with open('requests/__init__.py', 'r') as fd: version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', @@ -88,6 +88,7 @@ setup( 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy' ), diff --git a/tests/test_lowlevel.py b/tests/test_lowlevel.py index c17ba658..fbc2be22 100644 --- a/tests/test_lowlevel.py +++ b/tests/test_lowlevel.py @@ -5,7 +5,7 @@ import pytest import threading import requests -from tests.testserver.server import Server +from tests.testserver.server import Server, consume_socket_content from .utils import override_environ @@ -42,6 +42,154 @@ def test_incorrect_content_length(): close_server.set() # release server block +def test_digestauth_401_count_reset_on_redirect(): + """Ensure we correctly reset num_401_calls after a successful digest auth, + followed by a 302 redirect to another digest auth prompt. + + See https://github.com/kennethreitz/requests/issues/1979. + """ + text_401 = (b'HTTP/1.1 401 UNAUTHORIZED\r\n' + b'Content-Length: 0\r\n' + b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"' + b', opaque="372825293d1c26955496c80ed6426e9e", ' + b'realm="me@kennethreitz.com", qop=auth\r\n\r\n') + + text_302 = (b'HTTP/1.1 302 FOUND\r\n' + b'Content-Length: 0\r\n' + b'Location: /\r\n\r\n') + + text_200 = (b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 0\r\n\r\n') + + expected_digest = (b'Authorization: Digest username="user", ' + b'realm="me@kennethreitz.com", ' + b'nonce="6bf5d6e4da1ce66918800195d6b9130d", uri="/"') + + auth = requests.auth.HTTPDigestAuth('user', 'pass') + + def digest_response_handler(sock): + # Respond to initial GET with a challenge. + request_content = consume_socket_content(sock, timeout=0.5) + assert request_content.startswith(b"GET / HTTP/1.1") + sock.send(text_401) + + # Verify we receive an Authorization header in response, then redirect. + request_content = consume_socket_content(sock, timeout=0.5) + assert expected_digest in request_content + sock.send(text_302) + + # Verify Authorization isn't sent to the redirected host, + # then send another challenge. + request_content = consume_socket_content(sock, timeout=0.5) + assert b'Authorization:' not in request_content + sock.send(text_401) + + # Verify Authorization is sent correctly again, and return 200 OK. + request_content = consume_socket_content(sock, timeout=0.5) + assert expected_digest in request_content + sock.send(text_200) + + return request_content + + close_server = threading.Event() + server = Server(digest_response_handler, wait_to_close_event=close_server) + + with server as (host, port): + url = 'http://{0}:{1}/'.format(host, port) + r = requests.get(url, auth=auth) + # Verify server succeeded in authenticating. + assert r.status_code == 200 + # Verify Authorization was sent in final request. + assert 'Authorization' in r.request.headers + assert r.request.headers['Authorization'].startswith('Digest ') + # Verify redirect happened as we expected. + assert r.history[0].status_code == 302 + close_server.set() + + +def test_digestauth_401_only_sent_once(): + """Ensure we correctly respond to a 401 challenge once, and then + stop responding if challenged again. + """ + text_401 = (b'HTTP/1.1 401 UNAUTHORIZED\r\n' + b'Content-Length: 0\r\n' + b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"' + b', opaque="372825293d1c26955496c80ed6426e9e", ' + b'realm="me@kennethreitz.com", qop=auth\r\n\r\n') + + expected_digest = (b'Authorization: Digest username="user", ' + b'realm="me@kennethreitz.com", ' + b'nonce="6bf5d6e4da1ce66918800195d6b9130d", uri="/"') + + auth = requests.auth.HTTPDigestAuth('user', 'pass') + + def digest_failed_response_handler(sock): + # Respond to initial GET with a challenge. + request_content = consume_socket_content(sock, timeout=0.5) + assert request_content.startswith(b"GET / HTTP/1.1") + sock.send(text_401) + + # Verify we receive an Authorization header in response, then + # challenge again. + request_content = consume_socket_content(sock, timeout=0.5) + assert expected_digest in request_content + sock.send(text_401) + + # Verify the client didn't respond to second challenge. + request_content = consume_socket_content(sock, timeout=0.5) + assert request_content == b'' + + return request_content + + close_server = threading.Event() + server = Server(digest_failed_response_handler, wait_to_close_event=close_server) + + with server as (host, port): + url = 'http://{0}:{1}/'.format(host, port) + r = requests.get(url, auth=auth) + # Verify server didn't authenticate us. + assert r.status_code == 401 + assert r.history[0].status_code == 401 + close_server.set() + + +def test_digestauth_only_on_4xx(): + """Ensure we only send digestauth on 4xx challenges. + + See https://github.com/kennethreitz/requests/issues/3772. + """ + text_200_chal = (b'HTTP/1.1 200 OK\r\n' + b'Content-Length: 0\r\n' + b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"' + b', opaque="372825293d1c26955496c80ed6426e9e", ' + b'realm="me@kennethreitz.com", qop=auth\r\n\r\n') + + auth = requests.auth.HTTPDigestAuth('user', 'pass') + + def digest_response_handler(sock): + # Respond to GET with a 200 containing www-authenticate header. + request_content = consume_socket_content(sock, timeout=0.5) + assert request_content.startswith(b"GET / HTTP/1.1") + sock.send(text_200_chal) + + # Verify the client didn't respond with auth. + request_content = consume_socket_content(sock, timeout=0.5) + assert request_content == b'' + + return request_content + + close_server = threading.Event() + server = Server(digest_response_handler, wait_to_close_event=close_server) + + with server as (host, port): + url = 'http://{0}:{1}/'.format(host, port) + r = requests.get(url, auth=auth) + # Verify server didn't receive auth from us. + assert r.status_code == 200 + assert len(r.history) == 0 + close_server.set() + + _schemes_by_var_prefix = [ ('http', ['http']), ('https', ['https']), diff --git a/tests/test_requests.py b/tests/test_requests.py index 336f68dc..b059e199 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -21,7 +21,7 @@ from requests.compat import ( Morsel, cookielib, getproxies, str, urlparse, builtin_str, OrderedDict) from requests.cookies import ( - cookiejar_from_dict, morsel_to_cookie, merge_cookies) + cookiejar_from_dict, morsel_to_cookie) from requests.exceptions import ( ConnectionError, ConnectTimeout, InvalidScheme, InvalidURL, MissingScheme, ReadTimeout, Timeout, RetryError, TooManyRedirects, @@ -716,7 +716,7 @@ class TestRequests: post1 = requests.post(url, data={'some': 'data'}) assert post1.status_code == 200 - with open('requirements.txt') as f: + with open('Pipfile') as f: post2 = requests.post(url, files={'some': f}) assert post2.status_code == 200 @@ -776,7 +776,7 @@ class TestRequests: post1 = requests.post(url, data={'some': 'data'}) assert post1.status_code == 200 - with open('requirements.txt') as f: + with open('Pipfile') as f: post2 = requests.post(url, data={'some': 'data'}, files={'some': f}) assert post2.status_code == 200 @@ -813,7 +813,7 @@ class TestRequests: def test_conflicting_post_params(self, httpbin): url = httpbin('post') - with open('requirements.txt') as f: + with open('Pipfile') as f: pytest.raises(ValueError, "requests.post(url, data='[{\"some\": \"data\"}]', files={'some': f})") pytest.raises(ValueError, "requests.post(url, data=u('[{\"some\": \"data\"}]'), files={'some': f})") @@ -2495,4 +2495,3 @@ class TestPreparingURLs(object): r = requests.Request('GET', url=input, params=params) p = r.prepare() assert p.url == expected - diff --git a/tests/test_utils.py b/tests/test_utils.py index c92c7323..1edf6218 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,7 +4,7 @@ from io import BytesIO import pytest from requests import compat -from requests.cookies import RequestsCookieJar, cookiejar_from_dict +from requests.cookies import RequestsCookieJar from requests.structures import CaseInsensitiveDict from requests.utils import ( address_in_network, dotted_netmask, @@ -274,6 +274,17 @@ class TestGuessJSONUTF: def test_bad_utf_like_encoding(self): assert guess_json_utf(b'\x00\x00\x00\x00') is None + @pytest.mark.parametrize( + ('encoding', 'expected'), ( + ('utf-16-be', 'utf-16'), + ('utf-16-le', 'utf-16'), + ('utf-32-be', 'utf-32'), + ('utf-32-le', 'utf-32') + )) + def test_guess_by_bom(self, encoding, expected): + data = u'\ufeff{}'.encode(encoding) + assert guess_json_utf(data) == expected + USER = PASSWORD = "%!*'();:@&=+$,/?#[] " ENCODED_USER = compat.quote(USER, '')