diff --git a/.gitignore b/.gitignore index 6e54ee34..ab967877 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,7 @@ dist /.mypy_cache/ .vscode/ + +/.pytest_cache/ + +/.tox/ diff --git a/.travis.yml b/.travis.yml index 53833fb8..d7685acc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,19 @@ language: python +python: + # - "3.4" + # - "3.5" + - "3.6" + # - "3.7-dev" + # - "pypy" -- appears to hang + # - "pypy3" +matrix: + allow_failures: + - python: 3.7-dev # command to install dependencies install: "make" # command to run tests script: - - | - if [[ "$TRAVIS_PYTHON_VERSION" != "2.6" ]] ; then make test-readme; fi + - make test-readme - make ci cache: pip jobs: @@ -34,7 +43,7 @@ jobs: - make test-readme - make ci python: '3.7' - dist: xenial + dist: xenial - stage: coverage python: '3.6' script: codecov diff --git a/3.0-HISTORY.rst b/3.0-HISTORY.rst new file mode 100644 index 00000000..f9b1ec07 --- /dev/null +++ b/3.0-HISTORY.rst @@ -0,0 +1,81 @@ +3.0.0 (2017-xx-xx) +++++++++++++++++++ + +- Support for Python 2.6 has been dropped. + + - The ``OrderedDict`` import no longer exists in compat.py because it is part + of ``collections`` in Python 2.7 and newer. + +- Simplified logic for determining Content-Length and Transfer-Encoding. + Requests will now avoid setting both headers on the same request, and + raise an exception if this is done manually by a user. + +- Remove the HTTPProxyAuth class in favor of supporting proxy auth via + the proxies parameter. + +- Relax how Requests strips bodies from redirects. 3.0.0 only supports body + removal on 301/302 POST redirects and all 303 redirects. + +- Remove support for non-string/bytes parameters in ``_basic_auth_str``. + +- Prevent ``Session.merge_environment`` from erroneously setting the + ``verify`` parameter to ``None`` instead of ``True``. + +- Streaming responses with ``Response.iter_lines`` or ``Response.iter_content`` + now requires an encoding to be set if one isn't provided by the server. + +- Exception raised during read timeout for ``Response.iter_content`` and + ``Response.iter_lines`` changed from ``ConnectionError`` to more + specific ``ReadTimeout``. + +- Raise exception if multiple locations are returned during a redirect. + +- Update ConnectionPool connections when TLS/SSL settings change. + +- Remove simplejson import and only use standard json module. + +- Strip surrounding whitespace from urls. + +- MissingSchema and InvalidSchema renamed to MissingScheme and InvalidScheme + respectively. + +- Change merge order for environment settings to avoid excluding Session-level + settings. + +- Encode redirect URIs as latin-1 before performing redirects in Python 3 to + avoid mangling during the requoting process. + +- Remove the ``__bool__`` and ``__nonzero__`` methods from a ``Response`` + object. + + This has been a planned feature for over a year. The behaviour is surprising + to most people and breaks most of the assumptions that people have about + Response objects. This resolves issue `#2002`_ + +- Skip over empty chunks in iterators. Empty chunks could prematurely signal + the end of a request body's transmission, skipping them allows all of the + data through. See `#2631`_ for more details. + +- Rename the ``req`` argument from ``Session.resolve_redirects`` method + to ``request``. + +- Rename the ``resp`` argument from ``Session.resolve_redirects`` to + ``response``. + +- New ``PreparedRequest.send`` method. Now, you can + ``Request().prepare().send()``. + +- All porcelain API functions (e.g. ``requests.get``, etc) now accept an + optional ``session`` parameter. If provided, the session given will be used + for the request, in place of one being created for you. + +- URLs are now automatically stripped of leading/trailing whitespace. + +- ``Response.raise_for_status()`` now returns the response object for good responses + +- Use ``HTTPHeaderDict`` for response headers, allowing easier access to + individual values when multiple response headers are sent using the same + header name. + +.. _#2002: https://github.com/kennethreitz/requests/issues/2002 +.. _#2631: https://github.com/kennethreitz/requests/issues/2631 diff --git a/AUTHORS.rst b/AUTHORS.rst index f0ee696b..0e9af363 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -157,16 +157,19 @@ Patches and Suggestions - Muhammad Yasoob Ullah Khalid (`@yasoob `_) - Paul van der Linden (`@pvanderlinden `_) - Colin Dickson (`@colindickson `_) +- Sabari Kumar Murugesan (`@neosab `_) - Smiley Barry (`@smiley `_) - Shagun Sodhani (`@shagunsodhani `_) - Robin Linderborg (`@vienno `_) - Brian Samek (`@bsamek `_) - Dmitry Dygalo (`@Stranger6667 `_) +- Tomáš Heger (`@geckon `_) - piotrjurkiewicz - Jesse Shapiro (`@haikuginger `_) - Nate Prewitt (`@nateprewitt `_) - Maik Himstedt - Michael Hunsinger +- Jeremy Cline (`@jeremycline `_) - Brian Bamsch (`@bbamsch `_) - Om Prakash Kumar (`@iamprakashom `_) - Philipp Konrad (`@gardiac2002 `_) @@ -180,12 +183,16 @@ Patches and Suggestions - Shmuel Amar (`@shmuelamar `_) - Gary Wu (`@garywu `_) - Ryan Pineo (`@ryanpineo `_) +<<<<<<< HEAD - Ed Morley (`@edmorley `_) - Matt Liu (`@mlcrazy `_) - Taylor Hoff (`@PrimordialHelios `_) +<<<<<<< HEAD - Arthur Vigil (`@ahvigil `_) - Nehal J Wani (`@nehaljwani `_) - Demetrios Bairaktaris (`@DemetriosBairaktaris `_) - Darren Dormer (`@ddormer `_) - Rajiv Mayani (`@mayani `_) - Antti Kaihola (`@akaihola `_) +- Hugo van Kemenade (`@hugovk `_) +- Allan Crooks (`@the-allanc `_) diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 00000000..36ebe1c6 --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,1545 @@ +.. :changelog: + +Release History +--------------- + +dev ++++ + +**Improvements** + +- Warn user about possible slowdown when using cryptography version < 1.3.4 +- Use rfc3986 for URL parsing. +- Use __slots__ for often–called classes. + +**Bugfixes** + +- Parsing empty ``Link`` headers with ``parse_header_links()`` no longer return one bogus entry + +2.18.4 (2017-08-15) ++++++++++++++++++++ + +**Improvements** + +- Error messages for invalid headers now include the header name for easier debugging + +**Dependencies** + +- We now support idna v2.6. + +2.18.3 (2017-08-02) ++++++++++++++++++++ + +**Improvements** + +- Running ``$ python -m requests.help`` now includes the installed version of idna. + +**Bugfixes** + +- Fixed issue where Requests would raise ``ConnectionError`` instead of + ``SSLError`` when encountering SSL problems when using urllib3 v1.22. + +2.18.2 (2017-07-25) ++++++++++++++++++++ + +**Bugfixes** + +- ``requests.help`` no longer fails on Python 2.6 due to the absence of + ``ssl.OPENSSL_VERSION_NUMBER``. + +**Dependencies** + +- We now support urllib3 v1.22. + +2.18.1 (2017-06-14) ++++++++++++++++++++ + +**Bugfixes** + +- Fix an error in the packaging whereby the ``*.whl`` contained incorrect data + that regressed the fix in v2.17.3. + +2.18.0 (2017-06-14) ++++++++++++++++++++ + +**Improvements** + +- ``Response`` is now a context manager, so can be used directly in a ``with`` statement + without first having to be wrapped by ``contextlib.closing()``. + +**Bugfixes** + +- Resolve installation failure if multiprocessing is not available +- Resolve tests crash if multiprocessing is not able to determine the number of CPU cores +- Resolve error swallowing in utils set_environ generator + + +2.17.3 (2017-05-29) ++++++++++++++++++++ + +**Improvements** + +- Improved ``packages`` namespace identity support, for monkeypatching libraries. + + +2.17.2 (2017-05-29) ++++++++++++++++++++ + +**Improvements** + +- Improved ``packages`` namespace identity support, for monkeypatching libraries. + + +2.17.1 (2017-05-29) ++++++++++++++++++++ + +**Improvements** + +- Improved ``packages`` namespace identity support, for monkeypatching libraries. + + +2.17.0 (2017-05-29) ++++++++++++++++++++ + +**Improvements** + +- Removal of the 301 redirect cache. This improves thread-safety. + + +2.16.5 (2017-05-28) ++++++++++++++++++++ + +- Improvements to ``$ python -m requests.help``. + +2.16.4 (2017-05-27) ++++++++++++++++++++ + +- Introduction of the ``$ python -m requests.help`` command, for debugging with maintainers! + +2.16.3 (2017-05-27) ++++++++++++++++++++ + +- Further restored the ``requests.packages`` namespace for compatibility reasons. + +2.16.2 (2017-05-27) ++++++++++++++++++++ + +- Further restored the ``requests.packages`` namespace for compatibility reasons. + +No code modification (noted below) should be neccessary any longer. + +2.16.1 (2017-05-27) ++++++++++++++++++++ + +- Restored the ``requests.packages`` namespace for compatibility reasons. +- Bugfix for ``urllib3`` version parsing. + +**Note**: code that was written to import against the ``requests.packages`` +namespace previously will have to import code that rests at this module-level +now. + +For example:: + + from requests.packages.urllib3.poolmanager import PoolManager + +Will need to be re-written to be:: + + from requests.packages import urllib3 + urllib3.poolmanager.PoolManager + +Or, even better:: + + from urllib3.poolmanager import PoolManager + +2.16.0 (2017-05-26) ++++++++++++++++++++ + +- Unvendor ALL the things! + +2.15.1 (2017-05-26) ++++++++++++++++++++ + +- Everyone makes mistakes. + +2.15.0 (2017-05-26) ++++++++++++++++++++ + +**Improvements** + +- Introduction of the ``Response.next`` property, for getting the next + ``PreparedResponse`` from a redirect chain (when ``allow_redirects=False``). +- Internal refactoring of ``__version__`` module. + +**Bugfixes** + +- Restored once-optional parameter for ``requests.utils.get_environ_proxies()``. + +2.14.2 (2017-05-10) ++++++++++++++++++++ + +**Bugfixes** + +- Changed a less-than to an equal-to and an or in the dependency markers to + widen compatibility with older setuptools releases. + +2.14.1 (2017-05-09) ++++++++++++++++++++ + +**Bugfixes** + +- Changed the dependency markers to widen compatibility with older pip + releases. + +2.14.0 (2017-05-09) ++++++++++++++++++++ + +**Improvements** + +- It is now possible to pass ``no_proxy`` as a key to the ``proxies`` + dictionary to provide handling similar to the ``NO_PROXY`` environment + variable. +- When users provide invalid paths to certificate bundle files or directories + Requests now raises ``IOError``, rather than failing at the time of the HTTPS + request with a fairly inscrutable certificate validation error. +- The behavior of ``SessionRedirectMixin`` was slightly altered. + ``resolve_redirects`` will now detect a redirect by calling + ``get_redirect_target(response)`` instead of directly + querying ``Response.is_redirect`` and ``Response.headers['location']``. + Advanced users will be able to process malformed redirects more easily. +- Changed the internal calculation of elapsed request time to have higher + resolution on Windows. +- Added ``win_inet_pton`` as conditional dependency for the ``[socks]`` extra + on Windows with Python 2.7. +- Changed the proxy bypass implementation on Windows: the proxy bypass + check doesn't use forward and reverse DNS requests anymore +- URLs with schemes that begin with ``http`` but are not ``http`` or ``https`` + no longer have their host parts forced to lowercase. + +**Bugfixes** + +- Much improved handling of non-ASCII ``Location`` header values in redirects. + Fewer ``UnicodeDecodeErrors`` are encountered on Python 2, and Python 3 now + correctly understands that Latin-1 is unlikely to be the correct encoding. +- If an attempt to ``seek`` file to find out its length fails, we now + appropriately handle that by aborting our content-length calculations. +- Restricted ``HTTPDigestAuth`` to only respond to auth challenges made on 4XX + responses, rather than to all auth challenges. +- Fixed some code that was firing ``DeprecationWarning`` on Python 3.6. +- The dismayed person emoticon (``/o\\``) no longer has a big head. I'm sure + this is what you were all worrying about most. + + +**Miscellaneous** + +- Updated bundled urllib3 to v1.21.1. +- Updated bundled chardet to v3.0.2. +- Updated bundled idna to v2.5. +- Updated bundled certifi to 2017.4.17. + +- Altered how ``SessionRedirectMixin.resolve_redirects`` and ``Session.send`` + process redirect history. Developers who subclass ``resolve_redirects`` will + find a different ``.history`` attribute - the first element now contains the + original response, and the last element now contains the active response. + +2.13.0 (2017-01-24) ++++++++++++++++++++ + +**Features** + +- Only load the ``idna`` library when we've determined we need it. This will + save some memory for users. + +**Miscellaneous** + +- Updated bundled urllib3 to 1.20. +- Updated bundled idna to 2.2. + +2.12.5 (2017-01-18) ++++++++++++++++++++ + +**Bugfixes** + +- Fixed an issue with JSON encoding detection, specifically detecting + big-endian UTF-32 with BOM. + +2.12.4 (2016-12-14) ++++++++++++++++++++ + +**Bugfixes** + +- Fixed regression from 2.12.2 where non-string types were rejected in the + basic auth parameters. While support for this behaviour has been readded, + the behaviour is deprecated and will be removed in the future. + +2.12.3 (2016-12-01) ++++++++++++++++++++ + +**Bugfixes** + +- Fixed regression from v2.12.1 for URLs with schemes that begin with "http". + These URLs have historically been processed as though they were HTTP-schemed + URLs, and so have had parameters added. This was removed in v2.12.2 in an + overzealous attempt to resolve problems with IDNA-encoding those URLs. This + change was reverted: the other fixes for IDNA-encoding have been judged to + be sufficient to return to the behaviour Requests had before v2.12.0. + +2.12.2 (2016-11-30) ++++++++++++++++++++ + +**Bugfixes** + +- Fixed several issues with IDNA-encoding URLs that are technically invalid but + which are widely accepted. Requests will now attempt to IDNA-encode a URL if + it can but, if it fails, and the host contains only ASCII characters, it will + be passed through optimistically. This will allow users to opt-in to using + IDNA2003 themselves if they want to, and will also allow technically invalid + but still common hostnames. +- Fixed an issue where URLs with leading whitespace would raise + ``InvalidSchema`` errors. +- Fixed an issue where some URLs without the HTTP or HTTPS schemes would still + have HTTP URL preparation applied to them. +- Fixed an issue where Unicode strings could not be used in basic auth. +- Fixed an issue encountered by some Requests plugins where constructing a + Response object would cause ``Response.content`` to raise an + ``AttributeError``. + +2.12.1 (2016-11-16) ++++++++++++++++++++ + +**Bugfixes** + +- Updated setuptools 'security' extra for the new PyOpenSSL backend in urllib3. + +**Miscellaneous** + +- Updated bundled urllib3 to 1.19.1. + +2.12.0 (2016-11-15) ++++++++++++++++++++ + +**Improvements** + +- Updated support for internationalized domain names from IDNA2003 to IDNA2008. + This updated support is required for several forms of IDNs and is mandatory + for .de domains. +- Much improved heuristics for guessing content lengths: Requests will no + longer read an entire ``StringIO`` into memory. +- Much improved logic for recalculating ``Content-Length`` headers for + ``PreparedRequest`` objects. +- Improved tolerance for file-like objects that have no ``tell`` method but + do have a ``seek`` method. +- Anything that is a subclass of ``Mapping`` is now treated like a dictionary + by the ``data=`` keyword argument. +- Requests now tolerates empty passwords in proxy credentials, rather than + stripping the credentials. +- If a request is made with a file-like object as the body and that request is + redirected with a 307 or 308 status code, Requests will now attempt to + rewind the body object so it can be replayed. + +**Bugfixes** + +- When calling ``response.close``, the call to ``close`` will be propagated + through to non-urllib3 backends. +- Fixed issue where the ``ALL_PROXY`` environment variable would be preferred + over scheme-specific variables like ``HTTP_PROXY``. +- Fixed issue where non-UTF8 reason phrases got severely mangled by falling + back to decoding using ISO 8859-1 instead. +- Fixed a bug where Requests would not correctly correlate cookies set when + using custom Host headers if those Host headers did not use the native + string type for the platform. + +**Miscellaneous** + +- Updated bundled urllib3 to 1.19. +- Updated bundled certifi certs to 2016.09.26. + +2.11.1 (2016-08-17) ++++++++++++++++++++ + +**Bugfixes** + +- Fixed a bug when using ``iter_content`` with ``decode_unicode=True`` for + streamed bodies would raise ``AttributeError``. This bug was introduced in + 2.11. +- Strip Content-Type and Transfer-Encoding headers from the header block when + following a redirect that transforms the verb from POST/PUT to GET. + +2.11.0 (2016-08-08) ++++++++++++++++++++ + +**Improvements** + +- Added support for the ``ALL_PROXY`` environment variable. +- Reject header values that contain leading whitespace or newline characters to + reduce risk of header smuggling. + +**Bugfixes** + +- Fixed occasional ``TypeError`` when attempting to decode a JSON response that + occurred in an error case. Now correctly returns a ``ValueError``. +- Requests would incorrectly ignore a non-CIDR IP address in the ``NO_PROXY`` + environment variables: Requests now treats it as a specific IP. +- Fixed a bug when sending JSON data that could cause us to encounter obscure + OpenSSL errors in certain network conditions (yes, really). +- Added type checks to ensure that ``iter_content`` only accepts integers and + ``None`` for chunk sizes. +- Fixed issue where responses whose body had not been fully consumed would have + the underlying connection closed but not returned to the connection pool, + which could cause Requests to hang in situations where the ``HTTPAdapter`` + had been configured to use a blocking connection pool. + +**Miscellaneous** + +- Updated bundled urllib3 to 1.16. +- Some previous releases accidentally accepted non-strings as acceptable header values. This release does not. + +2.10.0 (2016-04-29) ++++++++++++++++++++ + +**New Features** + +- SOCKS Proxy Support! (requires PySocks; ``$ pip install requests[socks]``) + +**Miscellaneous** + +- Updated bundled urllib3 to 1.15.1. + +2.9.2 (2016-04-29) +++++++++++++++++++ + +**Improvements** + +- Change built-in CaseInsensitiveDict (used for headers) to use OrderedDict + as its underlying datastore. + +**Bugfixes** + +- Don't use redirect_cache if allow_redirects=False +- When passed objects that throw exceptions from ``tell()``, send them via + chunked transfer encoding instead of failing. +- Raise a ProxyError for proxy related connection issues. + +2.9.1 (2015-12-21) +++++++++++++++++++ + +**Bugfixes** + +- Resolve regression introduced in 2.9.0 that made it impossible to send binary + strings as bodies in Python 3. +- Fixed errors when calculating cookie expiration dates in certain locales. + +**Miscellaneous** + +- Updated bundled urllib3 to 1.13.1. + +2.9.0 (2015-12-15) +++++++++++++++++++ + +**Minor Improvements** (Backwards compatible) + +- The ``verify`` keyword argument now supports being passed a path to a + directory of CA certificates, not just a single-file bundle. +- Warnings are now emitted when sending files opened in text mode. +- Added the 511 Network Authentication Required status code to the status code + registry. + +**Bugfixes** + +- For file-like objects that are not seeked to the very beginning, we now + send the content length for the number of bytes we will actually read, rather + than the total size of the file, allowing partial file uploads. +- When uploading file-like objects, if they are empty or have no obvious + content length we set ``Transfer-Encoding: chunked`` rather than + ``Content-Length: 0``. +- We correctly receive the response in buffered mode when uploading chunked + bodies. +- We now handle being passed a query string as a bytestring on Python 3, by + decoding it as UTF-8. +- Sessions are now closed in all cases (exceptional and not) when using the + functional API rather than leaking and waiting for the garbage collector to + clean them up. +- Correctly handle digest auth headers with a malformed ``qop`` directive that + contains no token, by treating it the same as if no ``qop`` directive was + provided at all. +- Minor performance improvements when removing specific cookies by name. + +**Miscellaneous** + +- Updated urllib3 to 1.13. + +2.8.1 (2015-10-13) +++++++++++++++++++ + +**Bugfixes** + +- Update certificate bundle to match ``certifi`` 2015.9.6.2's weak certificate + bundle. +- Fix a bug in 2.8.0 where requests would raise ``ConnectTimeout`` instead of + ``ConnectionError`` +- When using the PreparedRequest flow, requests will now correctly respect the + ``json`` parameter. Broken in 2.8.0. +- When using the PreparedRequest flow, requests will now correctly handle a + Unicode-string method name on Python 2. Broken in 2.8.0. + +2.8.0 (2015-10-05) +++++++++++++++++++ + +**Minor Improvements** (Backwards Compatible) + +- Requests now supports per-host proxies. This allows the ``proxies`` + dictionary to have entries of the form + ``{'://': ''}``. Host-specific proxies will be used + in preference to the previously-supported scheme-specific ones, but the + previous syntax will continue to work. +- ``Response.raise_for_status`` now prints the URL that failed as part of the + exception message. +- ``requests.utils.get_netrc_auth`` now takes an ``raise_errors`` kwarg, + defaulting to ``False``. When ``True``, errors parsing ``.netrc`` files cause + exceptions to be thrown. +- Change to bundled projects import logic to make it easier to unbundle + requests downstream. +- Changed the default User-Agent string to avoid leaking data on Linux: now + contains only the requests version. + +**Bugfixes** + +- The ``json`` parameter to ``post()`` and friends will now only be used if + neither ``data`` nor ``files`` are present, consistent with the + documentation. +- We now ignore empty fields in the ``NO_PROXY`` environment variable. +- Fixed problem where ``httplib.BadStatusLine`` would get raised if combining + ``stream=True`` with ``contextlib.closing``. +- Prevented bugs where we would attempt to return the same connection back to + the connection pool twice when sending a Chunked body. +- Miscellaneous minor internal changes. +- Digest Auth support is now thread safe. + +**Updates** + +- Updated urllib3 to 1.12. + +2.7.0 (2015-05-03) +++++++++++++++++++ + +This is the first release that follows our new release process. For more, see +`our documentation +`_. + +**Bugfixes** + +- Updated urllib3 to 1.10.4, resolving several bugs involving chunked transfer + encoding and response framing. + +2.6.2 (2015-04-23) +++++++++++++++++++ + +**Bugfixes** + +- Fix regression where compressed data that was sent as chunked data was not + properly decompressed. (#2561) + +2.6.1 (2015-04-22) +++++++++++++++++++ + +**Bugfixes** + +- Remove VendorAlias import machinery introduced in v2.5.2. + +- Simplify the PreparedRequest.prepare API: We no longer require the user to + pass an empty list to the hooks keyword argument. (c.f. #2552) + +- Resolve redirects now receives and forwards all of the original arguments to + the adapter. (#2503) + +- Handle UnicodeDecodeErrors when trying to deal with a unicode URL that + cannot be encoded in ASCII. (#2540) + +- Populate the parsed path of the URI field when performing Digest + Authentication. (#2426) + +- Copy a PreparedRequest's CookieJar more reliably when it is not an instance + of RequestsCookieJar. (#2527) + +2.6.0 (2015-03-14) +++++++++++++++++++ + +**Bugfixes** + +- CVE-2015-2296: Fix handling of cookies on redirect. Previously a cookie + without a host value set would use the hostname for the redirected URL + exposing requests users to session fixation attacks and potentially cookie + stealing. This was disclosed privately by Matthew Daley of + `BugFuzz `_. This affects all versions of requests from + v2.1.0 to v2.5.3 (inclusive on both ends). + +- Fix error when requests is an ``install_requires`` dependency and ``python + setup.py test`` is run. (#2462) + +- Fix error when urllib3 is unbundled and requests continues to use the + vendored import location. + +- Include fixes to ``urllib3``'s header handling. + +- Requests' handling of unvendored dependencies is now more restrictive. + +**Features and Improvements** + +- Support bytearrays when passed as parameters in the ``files`` argument. + (#2468) + +- Avoid data duplication when creating a request with ``str``, ``bytes``, or + ``bytearray`` input to the ``files`` argument. + +2.5.3 (2015-02-24) +++++++++++++++++++ + +**Bugfixes** + +- Revert changes to our vendored certificate bundle. For more context see + (#2455, #2456, and http://bugs.python.org/issue23476) + +2.5.2 (2015-02-23) +++++++++++++++++++ + +**Features and Improvements** + +- Add sha256 fingerprint support. (`shazow/urllib3#540`_) + +- Improve the performance of headers. (`shazow/urllib3#544`_) + +**Bugfixes** + +- Copy pip's import machinery. When downstream redistributors remove + requests.packages.urllib3 the import machinery will continue to let those + same symbols work. Example usage in requests' documentation and 3rd-party + libraries relying on the vendored copies of urllib3 will work without having + to fallback to the system urllib3. + +- Attempt to quote parts of the URL on redirect if unquoting and then quoting + fails. (#2356) + +- Fix filename type check for multipart form-data uploads. (#2411) + +- Properly handle the case where a server issuing digest authentication + challenges provides both auth and auth-int qop-values. (#2408) + +- Fix a socket leak. (`shazow/urllib3#549`_) + +- Fix multiple ``Set-Cookie`` headers properly. (`shazow/urllib3#534`_) + +- Disable the built-in hostname verification. (`shazow/urllib3#526`_) + +- Fix the behaviour of decoding an exhausted stream. (`shazow/urllib3#535`_) + +**Security** + +- Pulled in an updated ``cacert.pem``. + +- Drop RC4 from the default cipher list. (`shazow/urllib3#551`_) + +.. _shazow/urllib3#551: https://github.com/shazow/urllib3/pull/551 +.. _shazow/urllib3#549: https://github.com/shazow/urllib3/pull/549 +.. _shazow/urllib3#544: https://github.com/shazow/urllib3/pull/544 +.. _shazow/urllib3#540: https://github.com/shazow/urllib3/pull/540 +.. _shazow/urllib3#535: https://github.com/shazow/urllib3/pull/535 +.. _shazow/urllib3#534: https://github.com/shazow/urllib3/pull/534 +.. _shazow/urllib3#526: https://github.com/shazow/urllib3/pull/526 + +2.5.1 (2014-12-23) +++++++++++++++++++ + +**Behavioural Changes** + +- Only catch HTTPErrors in raise_for_status (#2382) + +**Bugfixes** + +- Handle LocationParseError from urllib3 (#2344) +- Handle file-like object filenames that are not strings (#2379) +- Unbreak HTTPDigestAuth handler. Allow new nonces to be negotiated (#2389) + +2.5.0 (2014-12-01) +++++++++++++++++++ + +**Improvements** + +- Allow usage of urllib3's Retry object with HTTPAdapters (#2216) +- The ``iter_lines`` method on a response now accepts a delimiter with which + to split the content (#2295) + +**Behavioural Changes** + +- Add deprecation warnings to functions in requests.utils that will be removed + in 3.0 (#2309) +- Sessions used by the functional API are always closed (#2326) +- Restrict requests to HTTP/1.1 and HTTP/1.0 (stop accepting HTTP/0.9) (#2323) + +**Bugfixes** + +- Only parse the URL once (#2353) +- Allow Content-Length header to always be overridden (#2332) +- Properly handle files in HTTPDigestAuth (#2333) +- Cap redirect_cache size to prevent memory abuse (#2299) +- Fix HTTPDigestAuth handling of redirects after authenticating successfully + (#2253) +- Fix crash with custom method parameter to Session.request (#2317) +- Fix how Link headers are parsed using the regular expression library (#2271) + +**Documentation** + +- Add more references for interlinking (#2348) +- Update CSS for theme (#2290) +- Update width of buttons and sidebar (#2289) +- Replace references of Gittip with Gratipay (#2282) +- Add link to changelog in sidebar (#2273) + +2.4.3 (2014-10-06) +++++++++++++++++++ + +**Bugfixes** + +- Unicode URL improvements for Python 2. +- Re-order JSON param for backwards compat. +- Automatically defrag authentication schemes from host/pass URIs. (`#2249 `_) + + +2.4.2 (2014-10-05) +++++++++++++++++++ + +**Improvements** + +- FINALLY! Add json parameter for uploads! (`#2258 `_) +- Support for bytestring URLs on Python 3.x (`#2238 `_) + +**Bugfixes** + +- Avoid getting stuck in a loop (`#2244 `_) +- Multiple calls to iter* fail with unhelpful error. (`#2240 `_, `#2241 `_) + +**Documentation** + +- Correct redirection introduction (`#2245 `_) +- Added example of how to send multiple files in one request. (`#2227 `_) +- Clarify how to pass a custom set of CAs (`#2248 `_) + + + +2.4.1 (2014-09-09) +++++++++++++++++++ + +- Now has a "security" package extras set, ``$ pip install requests[security]`` +- Requests will now use Certifi if it is available. +- Capture and re-raise urllib3 ProtocolError +- Bugfix for responses that attempt to redirect to themselves forever (wtf?). + + +2.4.0 (2014-08-29) +++++++++++++++++++ + +**Behavioral Changes** + +- ``Connection: keep-alive`` header is now sent automatically. + +**Improvements** + +- Support for connect timeouts! Timeout now accepts a tuple (connect, read) which is used to set individual connect and read timeouts. +- Allow copying of PreparedRequests without headers/cookies. +- Updated bundled urllib3 version. +- Refactored settings loading from environment -- new `Session.merge_environment_settings`. +- Handle socket errors in iter_content. + + +2.3.0 (2014-05-16) +++++++++++++++++++ + +**API Changes** + +- New ``Response`` property ``is_redirect``, which is true when the + library could have processed this response as a redirection (whether + or not it actually did). +- The ``timeout`` parameter now affects requests with both ``stream=True`` and + ``stream=False`` equally. +- The change in v2.0.0 to mandate explicit proxy schemes has been reverted. + Proxy schemes now default to ``http://``. +- The ``CaseInsensitiveDict`` used for HTTP headers now behaves like a normal + dictionary when references as string or viewed in the interpreter. + +**Bugfixes** + +- No longer expose Authorization or Proxy-Authorization headers on redirect. + Fix CVE-2014-1829 and CVE-2014-1830 respectively. +- Authorization is re-evaluated each redirect. +- On redirect, pass url as native strings. +- Fall-back to autodetected encoding for JSON when Unicode detection fails. +- Headers set to ``None`` on the ``Session`` are now correctly not sent. +- Correctly honor ``decode_unicode`` even if it wasn't used earlier in the same + response. +- Stop advertising ``compress`` as a supported Content-Encoding. +- The ``Response.history`` parameter is now always a list. +- Many, many ``urllib3`` bugfixes. + +2.2.1 (2014-01-23) +++++++++++++++++++ + +**Bugfixes** + +- Fixes incorrect parsing of proxy credentials that contain a literal or encoded '#' character. +- Assorted urllib3 fixes. + +2.2.0 (2014-01-09) +++++++++++++++++++ + +**API Changes** + +- New exception: ``ContentDecodingError``. Raised instead of ``urllib3`` + ``DecodeError`` exceptions. + +**Bugfixes** + +- Avoid many many exceptions from the buggy implementation of ``proxy_bypass`` on OS X in Python 2.6. +- Avoid crashing when attempting to get authentication credentials from ~/.netrc when running as a user without a home directory. +- Use the correct pool size for pools of connections to proxies. +- Fix iteration of ``CookieJar`` objects. +- Ensure that cookies are persisted over redirect. +- Switch back to using chardet, since it has merged with charade. + +2.1.0 (2013-12-05) +++++++++++++++++++ + +- Updated CA Bundle, of course. +- Cookies set on individual Requests through a ``Session`` (e.g. via ``Session.get()``) are no longer persisted to the ``Session``. +- Clean up connections when we hit problems during chunked upload, rather than leaking them. +- Return connections to the pool when a chunked upload is successful, rather than leaking it. +- Match the HTTPbis recommendation for HTTP 301 redirects. +- Prevent hanging when using streaming uploads and Digest Auth when a 401 is received. +- Values of headers set by Requests are now always the native string type. +- Fix previously broken SNI support. +- Fix accessing HTTP proxies using proxy authentication. +- Unencode HTTP Basic usernames and passwords extracted from URLs. +- Support for IP address ranges for no_proxy environment variable +- Parse headers correctly when users override the default ``Host:`` header. +- Avoid munging the URL in case of case-sensitive servers. +- Looser URL handling for non-HTTP/HTTPS urls. +- Accept unicode methods in Python 2.6 and 2.7. +- More resilient cookie handling. +- Make ``Response`` objects pickleable. +- Actually added MD5-sess to Digest Auth instead of pretending to like last time. +- Updated internal urllib3. +- Fixed @Lukasa's lack of taste. + +2.0.1 (2013-10-24) +++++++++++++++++++ + +- Updated included CA Bundle with new mistrusts and automated process for the future +- Added MD5-sess to Digest Auth +- Accept per-file headers in multipart file POST messages. +- Fixed: Don't send the full URL on CONNECT messages. +- Fixed: Correctly lowercase a redirect scheme. +- Fixed: Cookies not persisted when set via functional API. +- Fixed: Translate urllib3 ProxyError into a requests ProxyError derived from ConnectionError. +- Updated internal urllib3 and chardet. + +2.0.0 (2013-09-24) +++++++++++++++++++ + +**API Changes:** + +- Keys in the Headers dictionary are now native strings on all Python versions, + i.e. bytestrings on Python 2, unicode on Python 3. +- Proxy URLs now *must* have an explicit scheme. A ``MissingSchema`` exception + will be raised if they don't. +- Timeouts now apply to read time if ``Stream=False``. +- ``RequestException`` is now a subclass of ``IOError``, not ``RuntimeError``. +- Added new method to ``PreparedRequest`` objects: ``PreparedRequest.copy()``. +- Added new method to ``Session`` objects: ``Session.update_request()``. This + method updates a ``Request`` object with the data (e.g. cookies) stored on + the ``Session``. +- Added new method to ``Session`` objects: ``Session.prepare_request()``. This + method updates and prepares a ``Request`` object, and returns the + corresponding ``PreparedRequest`` object. +- Added new method to ``HTTPAdapter`` objects: ``HTTPAdapter.proxy_headers()``. + This should not be called directly, but improves the subclass interface. +- ``httplib.IncompleteRead`` exceptions caused by incorrect chunked encoding + will now raise a Requests ``ChunkedEncodingError`` instead. +- Invalid percent-escape sequences now cause a Requests ``InvalidURL`` + exception to be raised. +- HTTP 208 no longer uses reason phrase ``"im_used"``. Correctly uses + ``"already_reported"``. +- HTTP 226 reason added (``"im_used"``). + +**Bugfixes:** + +- Vastly improved proxy support, including the CONNECT verb. Special thanks to + the many contributors who worked towards this improvement. +- Cookies are now properly managed when 401 authentication responses are + received. +- Chunked encoding fixes. +- Support for mixed case schemes. +- Better handling of streaming downloads. +- Retrieve environment proxies from more locations. +- Minor cookies fixes. +- Improved redirect behaviour. +- Improved streaming behaviour, particularly for compressed data. +- Miscellaneous small Python 3 text encoding bugs. +- ``.netrc`` no longer overrides explicit auth. +- Cookies set by hooks are now correctly persisted on Sessions. +- Fix problem with cookies that specify port numbers in their host field. +- ``BytesIO`` can be used to perform streaming uploads. +- More generous parsing of the ``no_proxy`` environment variable. +- Non-string objects can be passed in data values alongside files. + +1.2.3 (2013-05-25) +++++++++++++++++++ + +- Simple packaging fix + + +1.2.2 (2013-05-23) +++++++++++++++++++ + +- Simple packaging fix + + +1.2.1 (2013-05-20) +++++++++++++++++++ + +- 301 and 302 redirects now change the verb to GET for all verbs, not just + POST, improving browser compatibility. +- Python 3.3.2 compatibility +- Always percent-encode location headers +- Fix connection adapter matching to be most-specific first +- new argument to the default connection adapter for passing a block argument +- prevent a KeyError when there's no link headers + +1.2.0 (2013-03-31) +++++++++++++++++++ + +- Fixed cookies on sessions and on requests +- Significantly change how hooks are dispatched - hooks now receive all the + arguments specified by the user when making a request so hooks can make a + secondary request with the same parameters. This is especially necessary for + authentication handler authors +- certifi support was removed +- Fixed bug where using OAuth 1 with body ``signature_type`` sent no data +- Major proxy work thanks to @Lukasa including parsing of proxy authentication + from the proxy url +- Fix DigestAuth handling too many 401s +- Update vendored urllib3 to include SSL bug fixes +- Allow keyword arguments to be passed to ``json.loads()`` via the + ``Response.json()`` method +- Don't send ``Content-Length`` header by default on ``GET`` or ``HEAD`` + requests +- Add ``elapsed`` attribute to ``Response`` objects to time how long a request + took. +- Fix ``RequestsCookieJar`` +- Sessions and Adapters are now picklable, i.e., can be used with the + multiprocessing library +- Update charade to version 1.0.3 + +The change in how hooks are dispatched will likely cause a great deal of +issues. + +1.1.0 (2013-01-10) +++++++++++++++++++ + +- CHUNKED REQUESTS +- Support for iterable response bodies +- Assume servers persist redirect params +- Allow explicit content types to be specified for file data +- Make merge_kwargs case-insensitive when looking up keys + +1.0.3 (2012-12-18) +++++++++++++++++++ + +- Fix file upload encoding bug +- Fix cookie behavior + +1.0.2 (2012-12-17) +++++++++++++++++++ + +- Proxy fix for HTTPAdapter. + +1.0.1 (2012-12-17) +++++++++++++++++++ + +- Cert verification exception bug. +- Proxy fix for HTTPAdapter. + +1.0.0 (2012-12-17) +++++++++++++++++++ + +- Massive Refactor and Simplification +- Switch to Apache 2.0 license +- Swappable Connection Adapters +- Mountable Connection Adapters +- Mutable ProcessedRequest chain +- /s/prefetch/stream +- Removal of all configuration +- Standard library logging +- Make Response.json() callable, not property. +- Usage of new charade project, which provides python 2 and 3 simultaneous chardet. +- Removal of all hooks except 'response' +- Removal of all authentication helpers (OAuth, Kerberos) + +This is not a backwards compatible change. + +0.14.2 (2012-10-27) ++++++++++++++++++++ + +- Improved mime-compatible JSON handling +- Proxy fixes +- Path hack fixes +- Case-Insensitive Content-Encoding headers +- Support for CJK parameters in form posts + + +0.14.1 (2012-10-01) ++++++++++++++++++++ + +- Python 3.3 Compatibility +- Simply default accept-encoding +- Bugfixes + + +0.14.0 (2012-09-02) +++++++++++++++++++++ + +- No more iter_content errors if already downloaded. + +0.13.9 (2012-08-25) ++++++++++++++++++++ + +- Fix for OAuth + POSTs +- Remove exception eating from dispatch_hook +- General bugfixes + +0.13.8 (2012-08-21) ++++++++++++++++++++ + +- Incredible Link header support :) + +0.13.7 (2012-08-19) ++++++++++++++++++++ + +- Support for (key, value) lists everywhere. +- Digest Authentication improvements. +- Ensure proxy exclusions work properly. +- Clearer UnicodeError exceptions. +- Automatic casting of URLs to strings (fURL and such) +- Bugfixes. + +0.13.6 (2012-08-06) ++++++++++++++++++++ + +- Long awaited fix for hanging connections! + +0.13.5 (2012-07-27) ++++++++++++++++++++ + +- Packaging fix + +0.13.4 (2012-07-27) ++++++++++++++++++++ + +- GSSAPI/Kerberos authentication! +- App Engine 2.7 Fixes! +- Fix leaking connections (from urllib3 update) +- OAuthlib path hack fix +- OAuthlib URL parameters fix. + +0.13.3 (2012-07-12) ++++++++++++++++++++ + +- Use simplejson if available. +- Do not hide SSLErrors behind Timeouts. +- Fixed param handling with urls containing fragments. +- Significantly improved information in User Agent. +- client certificates are ignored when verify=False + +0.13.2 (2012-06-28) ++++++++++++++++++++ + +- Zero dependencies (once again)! +- New: Response.reason +- Sign querystring parameters in OAuth 1.0 +- Client certificates no longer ignored when verify=False +- Add openSUSE certificate support + +0.13.1 (2012-06-07) ++++++++++++++++++++ + +- Allow passing a file or file-like object as data. +- Allow hooks to return responses that indicate errors. +- Fix Response.text and Response.json for body-less responses. + +0.13.0 (2012-05-29) ++++++++++++++++++++ + +- Removal of Requests.async in favor of `grequests `_ +- Allow disabling of cookie persistence. +- New implementation of safe_mode +- cookies.get now supports default argument +- Session cookies not saved when Session.request is called with return_response=False +- Env: no_proxy support. +- RequestsCookieJar improvements. +- Various bug fixes. + +0.12.1 (2012-05-08) ++++++++++++++++++++ + +- New ``Response.json`` property. +- Ability to add string file uploads. +- Fix out-of-range issue with iter_lines. +- Fix iter_content default size. +- Fix POST redirects containing files. + +0.12.0 (2012-05-02) ++++++++++++++++++++ + +- EXPERIMENTAL OAUTH SUPPORT! +- Proper CookieJar-backed cookies interface with awesome dict-like interface. +- Speed fix for non-iterated content chunks. +- Move ``pre_request`` to a more usable place. +- New ``pre_send`` hook. +- Lazily encode data, params, files. +- Load system Certificate Bundle if ``certify`` isn't available. +- Cleanups, fixes. + +0.11.2 (2012-04-22) ++++++++++++++++++++ + +- Attempt to use the OS's certificate bundle if ``certifi`` isn't available. +- Infinite digest auth redirect fix. +- Multi-part file upload improvements. +- Fix decoding of invalid %encodings in URLs. +- If there is no content in a response don't throw an error the second time that content is attempted to be read. +- Upload data on redirects. + +0.11.1 (2012-03-30) ++++++++++++++++++++ + +* POST redirects now break RFC to do what browsers do: Follow up with a GET. +* New ``strict_mode`` configuration to disable new redirect behavior. + + +0.11.0 (2012-03-14) ++++++++++++++++++++ + +* Private SSL Certificate support +* Remove select.poll from Gevent monkeypatching +* Remove redundant generator for chunked transfer encoding +* Fix: Response.ok raises Timeout Exception in safe_mode + +0.10.8 (2012-03-09) ++++++++++++++++++++ + +* Generate chunked ValueError fix +* Proxy configuration by environment variables +* Simplification of iter_lines. +* New `trust_env` configuration for disabling system/environment hints. +* Suppress cookie errors. + +0.10.7 (2012-03-07) ++++++++++++++++++++ + +* `encode_uri` = False + +0.10.6 (2012-02-25) ++++++++++++++++++++ + +* Allow '=' in cookies. + +0.10.5 (2012-02-25) ++++++++++++++++++++ + +* Response body with 0 content-length fix. +* New async.imap. +* Don't fail on netrc. + + +0.10.4 (2012-02-20) ++++++++++++++++++++ + +* Honor netrc. + +0.10.3 (2012-02-20) ++++++++++++++++++++ + +* HEAD requests don't follow redirects anymore. +* raise_for_status() doesn't raise for 3xx anymore. +* Make Session objects picklable. +* ValueError for invalid schema URLs. + +0.10.2 (2012-01-15) ++++++++++++++++++++ + +* Vastly improved URL quoting. +* Additional allowed cookie key values. +* Attempted fix for "Too many open files" Error +* Replace unicode errors on first pass, no need for second pass. +* Append '/' to bare-domain urls before query insertion. +* Exceptions now inherit from RuntimeError. +* Binary uploads + auth fix. +* Bugfixes. + + +0.10.1 (2012-01-23) ++++++++++++++++++++ + +* PYTHON 3 SUPPORT! +* Dropped 2.5 Support. (*Backwards Incompatible*) + +0.10.0 (2012-01-21) ++++++++++++++++++++ + +* ``Response.content`` is now bytes-only. (*Backwards Incompatible*) +* New ``Response.text`` is unicode-only. +* If no ``Response.encoding`` is specified and ``chardet`` is available, ``Response.text`` will guess an encoding. +* Default to ISO-8859-1 (Western) encoding for "text" subtypes. +* Removal of `decode_unicode`. (*Backwards Incompatible*) +* New multiple-hooks system. +* New ``Response.register_hook`` for registering hooks within the pipeline. +* ``Response.url`` is now Unicode. + +0.9.3 (2012-01-18) +++++++++++++++++++ + +* SSL verify=False bugfix (apparent on windows machines). + +0.9.2 (2012-01-18) +++++++++++++++++++ + +* Asynchronous async.send method. +* Support for proper chunk streams with boundaries. +* session argument for Session classes. +* Print entire hook tracebacks, not just exception instance. +* Fix response.iter_lines from pending next line. +* Fix but in HTTP-digest auth w/ URI having query strings. +* Fix in Event Hooks section. +* Urllib3 update. + + +0.9.1 (2012-01-06) +++++++++++++++++++ + +* danger_mode for automatic Response.raise_for_status() +* Response.iter_lines refactor + +0.9.0 (2011-12-28) +++++++++++++++++++ + +* verify ssl is default. + + +0.8.9 (2011-12-28) +++++++++++++++++++ + +* Packaging fix. + + +0.8.8 (2011-12-28) +++++++++++++++++++ + +* SSL CERT VERIFICATION! +* Release of Cerifi: Mozilla's cert list. +* New 'verify' argument for SSL requests. +* Urllib3 update. + +0.8.7 (2011-12-24) +++++++++++++++++++ + +* iter_lines last-line truncation fix +* Force safe_mode for async requests +* Handle safe_mode exceptions more consistently +* Fix iteration on null responses in safe_mode + +0.8.6 (2011-12-18) +++++++++++++++++++ + +* Socket timeout fixes. +* Proxy Authorization support. + +0.8.5 (2011-12-14) +++++++++++++++++++ + +* Response.iter_lines! + +0.8.4 (2011-12-11) +++++++++++++++++++ + +* Prefetch bugfix. +* Added license to installed version. + +0.8.3 (2011-11-27) +++++++++++++++++++ + +* Converted auth system to use simpler callable objects. +* New session parameter to API methods. +* Display full URL while logging. + +0.8.2 (2011-11-19) +++++++++++++++++++ + +* New Unicode decoding system, based on over-ridable `Response.encoding`. +* Proper URL slash-quote handling. +* Cookies with ``[``, ``]``, and ``_`` allowed. + +0.8.1 (2011-11-15) +++++++++++++++++++ + +* URL Request path fix +* Proxy fix. +* Timeouts fix. + +0.8.0 (2011-11-13) +++++++++++++++++++ + +* Keep-alive support! +* Complete removal of Urllib2 +* Complete removal of Poster +* Complete removal of CookieJars +* New ConnectionError raising +* Safe_mode for error catching +* prefetch parameter for request methods +* OPTION method +* Async pool size throttling +* File uploads send real names +* Vendored in urllib3 + +0.7.6 (2011-11-07) +++++++++++++++++++ + +* Digest authentication bugfix (attach query data to path) + +0.7.5 (2011-11-04) +++++++++++++++++++ + +* Response.content = None if there was an invalid response. +* Redirection auth handling. + +0.7.4 (2011-10-26) +++++++++++++++++++ + +* Session Hooks fix. + +0.7.3 (2011-10-23) +++++++++++++++++++ + +* Digest Auth fix. + + +0.7.2 (2011-10-23) +++++++++++++++++++ + +* PATCH Fix. + + +0.7.1 (2011-10-23) +++++++++++++++++++ + +* Move away from urllib2 authentication handling. +* Fully Remove AuthManager, AuthObject, &c. +* New tuple-based auth system with handler callbacks. + + +0.7.0 (2011-10-22) +++++++++++++++++++ + +* Sessions are now the primary interface. +* Deprecated InvalidMethodException. +* PATCH fix. +* New config system (no more global settings). + + +0.6.6 (2011-10-19) +++++++++++++++++++ + +* Session parameter bugfix (params merging). + + +0.6.5 (2011-10-18) +++++++++++++++++++ + +* Offline (fast) test suite. +* Session dictionary argument merging. + + +0.6.4 (2011-10-13) +++++++++++++++++++ + +* Automatic decoding of unicode, based on HTTP Headers. +* New ``decode_unicode`` setting. +* Removal of ``r.read/close`` methods. +* New ``r.faw`` interface for advanced response usage.* +* Automatic expansion of parameterized headers. + + +0.6.3 (2011-10-13) +++++++++++++++++++ + +* Beautiful ``requests.async`` module, for making async requests w/ gevent. + + +0.6.2 (2011-10-09) +++++++++++++++++++ + +* GET/HEAD obeys allow_redirects=False. + + +0.6.1 (2011-08-20) +++++++++++++++++++ + +* Enhanced status codes experience ``\o/`` +* Set a maximum number of redirects (``settings.max_redirects``) +* Full Unicode URL support +* Support for protocol-less redirects. +* Allow for arbitrary request types. +* Bugfixes + + +0.6.0 (2011-08-17) +++++++++++++++++++ + +* New callback hook system +* New persistent sessions object and context manager +* Transparent Dict-cookie handling +* Status code reference object +* Removed Response.cached +* Added Response.request +* All args are kwargs +* Relative redirect support +* HTTPError handling improvements +* Improved https testing +* Bugfixes + + +0.5.1 (2011-07-23) +++++++++++++++++++ + +* International Domain Name Support! +* Access headers without fetching entire body (``read()``) +* Use lists as dicts for parameters +* Add Forced Basic Authentication +* Forced Basic is default authentication type +* ``python-requests.org`` default User-Agent header +* CaseInsensitiveDict lower-case caching +* Response.history bugfix + + +0.5.0 (2011-06-21) +++++++++++++++++++ + +* PATCH Support +* Support for Proxies +* HTTPBin Test Suite +* Redirect Fixes +* settings.verbose stream writing +* Querystrings for all methods +* URLErrors (Connection Refused, Timeout, Invalid URLs) are treated as explicitly raised + ``r.requests.get('hwe://blah'); r.raise_for_status()`` + + +0.4.1 (2011-05-22) +++++++++++++++++++ + +* Improved Redirection Handling +* New 'allow_redirects' param for following non-GET/HEAD Redirects +* Settings module refactoring + + +0.4.0 (2011-05-15) +++++++++++++++++++ + +* Response.history: list of redirected responses +* Case-Insensitive Header Dictionaries! +* Unicode URLs + + +0.3.4 (2011-05-14) +++++++++++++++++++ + +* Urllib2 HTTPAuthentication Recursion fix (Basic/Digest) +* Internal Refactor +* Bytes data upload Bugfix + + + +0.3.3 (2011-05-12) +++++++++++++++++++ + +* Request timeouts +* Unicode url-encoded data +* Settings context manager and module + + +0.3.2 (2011-04-15) +++++++++++++++++++ + +* Automatic Decompression of GZip Encoded Content +* AutoAuth Support for Tupled HTTP Auth + + +0.3.1 (2011-04-01) +++++++++++++++++++ + +* Cookie Changes +* Response.read() +* Poster fix + + +0.3.0 (2011-02-25) +++++++++++++++++++ + +* Automatic Authentication API Change +* Smarter Query URL Parameterization +* Allow file uploads and POST data together +* New Authentication Manager System + - Simpler Basic HTTP System + - Supports all build-in urllib2 Auths + - Allows for custom Auth Handlers + + +0.2.4 (2011-02-19) +++++++++++++++++++ + +* Python 2.5 Support +* PyPy-c v1.4 Support +* Auto-Authentication tests +* Improved Request object constructor + +0.2.3 (2011-02-15) +++++++++++++++++++ + +* New HTTPHandling Methods + - Response.__nonzero__ (false if bad HTTP Status) + - Response.ok (True if expected HTTP Status) + - Response.error (Logged HTTPError if bad HTTP Status) + - Response.raise_for_status() (Raises stored HTTPError) + + +0.2.2 (2011-02-14) +++++++++++++++++++ + +* Still handles request in the event of an HTTPError. (Issue #2) +* Eventlet and Gevent Monkeypatch support. +* Cookie Support (Issue #1) + + +0.2.1 (2011-02-14) +++++++++++++++++++ + +* Added file attribute to POST and PUT requests for multipart-encode file uploads. +* Added Request.url attribute for context and redirects + + +0.2.0 (2011-02-14) +++++++++++++++++++ + +* Birth! + + +0.0.1 (2011-02-13) +++++++++++++++++++ + +* Frustration +* Conception diff --git a/Makefile b/Makefile index 231ce357..de4d5345 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,18 @@ .PHONY: docs +core: + rm -fr requests3/core + git clone https://github.com/kennethreitz/requests-core + cd requests-core && python setup.py compile && cd .. + mv requests-core/requests_core requests3/core + rm -fr requests-core init: pip install pipenv --upgrade pipenv install --dev test: # This runs all of the tests, on both Python 2 and Python 3. - detox + python setup.py test +mypy: + python setup.py mypy ci: pipenv run py.test -n 8 --boxed --junitxml=report.xml diff --git a/Pipfile b/Pipfile index b6705a6c..43368bdd 100644 --- a/Pipfile +++ b/Pipfile @@ -1,8 +1,11 @@ [[source]] url = "https://pypi.org/simple/" + +url = "https://pypi.python.org/simple" verify_ssl = true name = "pypi" + [dev-packages] pytest = ">=2.8.0,<4.1" codecov = "*" @@ -22,3 +25,9 @@ httpbin = ">=0.7.0" [packages] "e1839a8" = {path = ".", editable = true, extras = ["socks"]} +httpbin = "==0.5.0" +pytest-mypy = {git = "https://github.com/petr-muller/pytest-mypy", editable = true, ref = "flush-errors"} +white = {version="*"} +mypy = "*" +"rfc3986" = "*" +twisted = {extras = ["tls"]} diff --git a/appveyor.yml b/appveyor.yml index 3b6cef63..5faf3e75 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,21 +5,6 @@ build: off environment: matrix: - - PYTHON: "C:\\Python27-x64" - PYTHON_VERSION: "2.7.x" - PYTHON_ARCH: "64" - TOXENV: "py27" - - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4.x" - PYTHON_ARCH: "64" - TOXENV: "py34" - - - PYTHON: "C:\\Python35-x64" - PYTHON_VERSION: "3.5.x" - PYTHON_ARCH: "64" - TOXENV: "py35" - - PYTHON: "C:\\Python36-x64" PYTHON_VERSION: "3.6.x" PYTHON_ARCH: "64" diff --git a/docs/_themes/flask_theme_support.py b/docs/_themes/flask_theme_support.py index 33f47449..814e8da2 100644 --- a/docs/_themes/flask_theme_support.py +++ b/docs/_themes/flask_theme_support.py @@ -1,86 +1,76 @@ # flasky extensions. flasky pygments style based on tango style from pygments.style import Style -from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic, Whitespace, Punctuation, Other, Literal +from pygments.token import Keyword, Name, Comment, String, Error, Number, Operator, Generic, Whitespace, Punctuation, Other, Literal class FlaskyStyle(Style): background_color = "#f8f8f8" default_style = "" - styles = { # No corresponding class for the following: #Text: "", # class: '' - Whitespace: "underline #f8f8f8", # class: 'w' - Error: "#a40000 border:#ef2929", # class: 'err' - Other: "#000000", # class 'x' - - Comment: "italic #8f5902", # class: 'c' - Comment.Preproc: "noitalic", # class: 'cp' - - Keyword: "bold #004461", # class: 'k' - Keyword.Constant: "bold #004461", # class: 'kc' - Keyword.Declaration: "bold #004461", # class: 'kd' - Keyword.Namespace: "bold #004461", # class: 'kn' - Keyword.Pseudo: "bold #004461", # class: 'kp' - Keyword.Reserved: "bold #004461", # class: 'kr' - Keyword.Type: "bold #004461", # class: 'kt' - - Operator: "#582800", # class: 'o' - Operator.Word: "bold #004461", # class: 'ow' - like keywords - - Punctuation: "bold #000000", # class: 'p' - + Whitespace: "underline #f8f8f8", # class: 'w' + Error: "#a40000 border:#ef2929", # class: 'err' + Other: "#000000", # class 'x' + Comment: "italic #8f5902", # class: 'c' + Comment.Preproc: "noitalic", # class: 'cp' + Keyword: "bold #004461", # class: 'k' + Keyword.Constant: "bold #004461", # class: 'kc' + Keyword.Declaration: "bold #004461", # class: 'kd' + Keyword.Namespace: "bold #004461", # class: 'kn' + Keyword.Pseudo: "bold #004461", # class: 'kp' + Keyword.Reserved: "bold #004461", # class: 'kr' + Keyword.Type: "bold #004461", # class: 'kt' + Operator: "#582800", # class: 'o' + Operator.Word: "bold #004461", # class: 'ow' - like keywords + Punctuation: "bold #000000", + # class: 'p' # because special names such as Name.Class, Name.Function, etc. # are not recognized as such later in the parsing, we choose them # to look the same as ordinary variables. - Name: "#000000", # class: 'n' - Name.Attribute: "#c4a000", # class: 'na' - to be revised - Name.Builtin: "#004461", # class: 'nb' - Name.Builtin.Pseudo: "#3465a4", # class: 'bp' - Name.Class: "#000000", # class: 'nc' - to be revised - Name.Constant: "#000000", # class: 'no' - to be revised - Name.Decorator: "#888", # class: 'nd' - to be revised - Name.Entity: "#ce5c00", # class: 'ni' - Name.Exception: "bold #cc0000", # class: 'ne' - Name.Function: "#000000", # class: 'nf' - Name.Property: "#000000", # class: 'py' - Name.Label: "#f57900", # class: 'nl' - Name.Namespace: "#000000", # class: 'nn' - to be revised - Name.Other: "#000000", # class: 'nx' - Name.Tag: "bold #004461", # class: 'nt' - like a keyword - Name.Variable: "#000000", # class: 'nv' - to be revised - Name.Variable.Class: "#000000", # class: 'vc' - to be revised - Name.Variable.Global: "#000000", # class: 'vg' - to be revised - Name.Variable.Instance: "#000000", # class: 'vi' - to be revised - - Number: "#990000", # class: 'm' - - Literal: "#000000", # class: 'l' - Literal.Date: "#000000", # class: 'ld' - - String: "#4e9a06", # class: 's' - String.Backtick: "#4e9a06", # class: 'sb' - String.Char: "#4e9a06", # class: 'sc' - String.Doc: "italic #8f5902", # class: 'sd' - like a comment - String.Double: "#4e9a06", # class: 's2' - String.Escape: "#4e9a06", # class: 'se' - String.Heredoc: "#4e9a06", # class: 'sh' - String.Interpol: "#4e9a06", # class: 'si' - String.Other: "#4e9a06", # class: 'sx' - String.Regex: "#4e9a06", # class: 'sr' - String.Single: "#4e9a06", # class: 's1' - String.Symbol: "#4e9a06", # class: 'ss' - - Generic: "#000000", # class: 'g' - Generic.Deleted: "#a40000", # class: 'gd' - Generic.Emph: "italic #000000", # class: 'ge' - Generic.Error: "#ef2929", # class: 'gr' - Generic.Heading: "bold #000080", # class: 'gh' - Generic.Inserted: "#00A000", # class: 'gi' - Generic.Output: "#888", # class: 'go' - Generic.Prompt: "#745334", # class: 'gp' - Generic.Strong: "bold #000000", # class: 'gs' - Generic.Subheading: "bold #800080", # class: 'gu' - Generic.Traceback: "bold #a40000", # class: 'gt' + Name: "#000000", # class: 'n' + Name.Attribute: "#c4a000", # class: 'na' - to be revised + Name.Builtin: "#004461", # class: 'nb' + Name.Builtin.Pseudo: "#3465a4", # class: 'bp' + Name.Class: "#000000", # class: 'nc' - to be revised + Name.Constant: "#000000", # class: 'no' - to be revised + Name.Decorator: "#888", # class: 'nd' - to be revised + Name.Entity: "#ce5c00", # class: 'ni' + Name.Exception: "bold #cc0000", # class: 'ne' + Name.Function: "#000000", # class: 'nf' + Name.Property: "#000000", # class: 'py' + Name.Label: "#f57900", # class: 'nl' + Name.Namespace: "#000000", # class: 'nn' - to be revised + Name.Other: "#000000", # class: 'nx' + Name.Tag: "bold #004461", # class: 'nt' - like a keyword + Name.Variable: "#000000", # class: 'nv' - to be revised + Name.Variable.Class: "#000000", # class: 'vc' - to be revised + Name.Variable.Global: "#000000", # class: 'vg' - to be revised + Name.Variable.Instance: "#000000", # class: 'vi' - to be revised + Number: "#990000", # class: 'm' + Literal: "#000000", # class: 'l' + Literal.Date: "#000000", # class: 'ld' + String: "#4e9a06", # class: 's' + String.Backtick: "#4e9a06", # class: 'sb' + String.Char: "#4e9a06", # class: 'sc' + String.Doc: "italic #8f5902", # class: 'sd' - like a comment + String.Double: "#4e9a06", # class: 's2' + String.Escape: "#4e9a06", # class: 'se' + String.Heredoc: "#4e9a06", # class: 'sh' + String.Interpol: "#4e9a06", # class: 'si' + String.Other: "#4e9a06", # class: 'sx' + String.Regex: "#4e9a06", # class: 'sr' + String.Single: "#4e9a06", # class: 's1' + String.Symbol: "#4e9a06", # class: 'ss' + Generic: "#000000", # class: 'g' + Generic.Deleted: "#a40000", # class: 'gd' + Generic.Emph: "italic #000000", # class: 'ge' + Generic.Error: "#ef2929", # class: 'gr' + Generic.Heading: "bold #000080", # class: 'gh' + Generic.Inserted: "#00A000", # class: 'gi' + Generic.Output: "#888", # class: 'go' + Generic.Prompt: "#745334", # class: 'gp' + Generic.Strong: "bold #000000", # class: 'gs' + Generic.Subheading: "bold #800080", # class: 'gu' + Generic.Traceback: "bold #a40000", # class: 'gt' } diff --git a/docs/api.rst b/docs/api.rst index 93cc4f0d..e9aabcc9 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -74,7 +74,6 @@ Authentication .. autoclass:: requests.auth.AuthBase .. autoclass:: requests.auth.HTTPBasicAuth -.. autoclass:: requests.auth.HTTPProxyAuth .. autoclass:: requests.auth.HTTPDigestAuth @@ -242,7 +241,7 @@ API Changes } # In requests 1.x, this was legal, in requests 2.x, - # this raises requests.exceptions.MissingSchema + # this raises requests.exceptions.MissingScheme requests.get("http://example.org", proxies=proxies) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index ca95a020..dce6c12c 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -388,10 +388,14 @@ But, since our ``status_code`` for ``r`` was ``200``, when we call ``raise_for_status()`` we get:: >>> r.raise_for_status() - None + All is well. +.. note:: ``raise_for_status`` returns the response object for a successful response. This eases chaining in trivial cases, where we want bad codes to raise an exception, but use the response otherwise: + + >>> value = requests.get('http://httpbin.org/ip').raise_for_status().json()['origin'] + Response Headers ---------------- @@ -431,6 +435,10 @@ represented in the dictionary within a single mapping, as per of the message, by appending each subsequent field value to the combined field value in order, separated by a comma. +If you do need to access each individual value sent with the same header, then +you can use the ``getlist`` method to get a sequence of all the values returned +for a particular header. + Cookies ------- diff --git a/requests/__version__.py b/requests/__version__.py deleted file mode 100644 index f5b5d036..00000000 --- a/requests/__version__.py +++ /dev/null @@ -1,14 +0,0 @@ -# .-. .-. .-. . . .-. .-. .-. .-. -# |( |- |.| | | |- `-. | `-. -# ' ' `-' `-`.`-' `-' `-' ' `-' - -__title__ = 'requests' -__description__ = 'Python HTTP for Humans.' -__url__ = 'http://python-requests.org' -__version__ = '2.21.0' -__build__ = 0x022100 -__author__ = 'Kenneth Reitz' -__author_email__ = 'me@kennethreitz.org' -__license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2018 Kenneth Reitz' -__cake__ = u'\u2728 \U0001f370 \u2728' diff --git a/requests/adapters.py b/requests/adapters.py deleted file mode 100644 index fa4d9b3c..00000000 --- a/requests/adapters.py +++ /dev/null @@ -1,533 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -requests.adapters -~~~~~~~~~~~~~~~~~ - -This module contains the transport adapters that Requests uses to define -and maintain connections. -""" - -import os.path -import socket - -from urllib3.poolmanager import PoolManager, proxy_from_url -from urllib3.response import HTTPResponse -from urllib3.util import parse_url -from urllib3.util import Timeout as TimeoutSauce -from urllib3.util.retry import Retry -from urllib3.exceptions import ClosedPoolError -from urllib3.exceptions import ConnectTimeoutError -from urllib3.exceptions import HTTPError as _HTTPError -from urllib3.exceptions import MaxRetryError -from urllib3.exceptions import NewConnectionError -from urllib3.exceptions import ProxyError as _ProxyError -from urllib3.exceptions import ProtocolError -from urllib3.exceptions import ReadTimeoutError -from urllib3.exceptions import SSLError as _SSLError -from urllib3.exceptions import ResponseError -from urllib3.exceptions import LocationValueError - -from .models import Response -from .compat import urlparse, basestring -from .utils import (DEFAULT_CA_BUNDLE_PATH, extract_zipped_paths, - get_encoding_from_headers, prepend_scheme_if_needed, - get_auth_from_url, urldefragauth, select_proxy) -from .structures import CaseInsensitiveDict -from .cookies import extract_cookies_to_jar -from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidSchema, InvalidProxyURL, - InvalidURL) -from .auth import _basic_auth_str - -try: - from urllib3.contrib.socks import SOCKSProxyManager -except ImportError: - def SOCKSProxyManager(*args, **kwargs): - raise InvalidSchema("Missing dependencies for SOCKS support.") - -DEFAULT_POOLBLOCK = False -DEFAULT_POOLSIZE = 10 -DEFAULT_RETRIES = 0 -DEFAULT_POOL_TIMEOUT = None - - -class BaseAdapter(object): - """The Base Transport Adapter""" - - def __init__(self): - super(BaseAdapter, self).__init__() - - def send(self, request, stream=False, timeout=None, verify=True, - cert=None, proxies=None): - """Sends PreparedRequest object. Returns Response object. - - :param request: The :class:`PreparedRequest ` being sent. - :param stream: (optional) Whether to stream the request content. - :param timeout: (optional) How long to wait for the server to send - data before giving up, as a float, or a :ref:`(connect timeout, - read timeout) ` tuple. - :type timeout: float or tuple - :param verify: (optional) Either a boolean, in which case it controls whether we verify - the server's TLS certificate, or a string, in which case it must be a path - to a CA bundle to use - :param cert: (optional) Any user-provided SSL certificate to be trusted. - :param proxies: (optional) The proxies dictionary to apply to the request. - """ - raise NotImplementedError - - def close(self): - """Cleans up adapter specific items.""" - raise NotImplementedError - - -class HTTPAdapter(BaseAdapter): - """The built-in HTTP Adapter for urllib3. - - Provides a general-case interface for Requests sessions to contact HTTP and - HTTPS urls by implementing the Transport Adapter interface. This class will - usually be created by the :class:`Session ` class under the - covers. - - :param pool_connections: The number of urllib3 connection pools to cache. - :param pool_maxsize: The maximum number of connections to save in the pool. - :param max_retries: The maximum number of retries each connection - should attempt. Note, this applies only to failed DNS lookups, socket - connections and connection timeouts, never to requests where data has - made it to the server. By default, Requests does not retry failed - connections. If you need granular control over the conditions under - which we retry a request, import urllib3's ``Retry`` class and pass - that instead. - :param pool_block: Whether the connection pool should block for connections. - - Usage:: - - >>> import requests - >>> s = requests.Session() - >>> a = requests.adapters.HTTPAdapter(max_retries=3) - >>> s.mount('http://', a) - """ - __attrs__ = ['max_retries', 'config', '_pool_connections', '_pool_maxsize', - '_pool_block'] - - def __init__(self, pool_connections=DEFAULT_POOLSIZE, - pool_maxsize=DEFAULT_POOLSIZE, max_retries=DEFAULT_RETRIES, - pool_block=DEFAULT_POOLBLOCK): - if max_retries == DEFAULT_RETRIES: - self.max_retries = Retry(0, read=False) - else: - self.max_retries = Retry.from_int(max_retries) - self.config = {} - self.proxy_manager = {} - - super(HTTPAdapter, self).__init__() - - self._pool_connections = pool_connections - self._pool_maxsize = pool_maxsize - self._pool_block = pool_block - - self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) - - def __getstate__(self): - return {attr: getattr(self, attr, None) for attr in self.__attrs__} - - def __setstate__(self, state): - # Can't handle by adding 'proxy_manager' to self.__attrs__ because - # self.poolmanager uses a lambda function, which isn't pickleable. - self.proxy_manager = {} - self.config = {} - - for attr, value in state.items(): - setattr(self, attr, value) - - self.init_poolmanager(self._pool_connections, self._pool_maxsize, - block=self._pool_block) - - def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs): - """Initializes a urllib3 PoolManager. - - This method should not be called from user code, and is only - exposed for use when subclassing the - :class:`HTTPAdapter `. - - :param connections: The number of urllib3 connection pools to cache. - :param maxsize: The maximum number of connections to save in the pool. - :param block: Block when no free connections are available. - :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager. - """ - # save these values for pickling - self._pool_connections = connections - self._pool_maxsize = maxsize - self._pool_block = block - - self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize, - block=block, strict=True, **pool_kwargs) - - def proxy_manager_for(self, proxy, **proxy_kwargs): - """Return urllib3 ProxyManager for the given proxy. - - This method should not be called from user code, and is only - exposed for use when subclassing the - :class:`HTTPAdapter `. - - :param proxy: The proxy to return a urllib3 ProxyManager for. - :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager. - :returns: ProxyManager - :rtype: urllib3.ProxyManager - """ - if proxy in self.proxy_manager: - manager = self.proxy_manager[proxy] - elif proxy.lower().startswith('socks'): - username, password = get_auth_from_url(proxy) - manager = self.proxy_manager[proxy] = SOCKSProxyManager( - proxy, - username=username, - password=password, - num_pools=self._pool_connections, - maxsize=self._pool_maxsize, - block=self._pool_block, - **proxy_kwargs - ) - else: - proxy_headers = self.proxy_headers(proxy) - manager = self.proxy_manager[proxy] = proxy_from_url( - proxy, - proxy_headers=proxy_headers, - num_pools=self._pool_connections, - maxsize=self._pool_maxsize, - block=self._pool_block, - **proxy_kwargs) - - return manager - - def cert_verify(self, conn, url, verify, cert): - """Verify a SSL certificate. This method should not be called from user - code, and is only exposed for use when subclassing the - :class:`HTTPAdapter `. - - :param conn: The urllib3 connection object associated with the cert. - :param url: The requested URL. - :param verify: Either a boolean, in which case it controls whether we verify - the server's TLS certificate, or a string, in which case it must be a path - to a CA bundle to use - :param cert: The SSL certificate to verify. - """ - if url.lower().startswith('https') and verify: - - cert_loc = None - - # Allow self-specified cert location. - if verify is not True: - cert_loc = verify - - if not cert_loc: - cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) - - if not cert_loc or not os.path.exists(cert_loc): - raise IOError("Could not find a suitable TLS CA certificate bundle, " - "invalid path: {}".format(cert_loc)) - - conn.cert_reqs = 'CERT_REQUIRED' - - if not os.path.isdir(cert_loc): - conn.ca_certs = cert_loc - else: - conn.ca_cert_dir = cert_loc - else: - conn.cert_reqs = 'CERT_NONE' - conn.ca_certs = None - conn.ca_cert_dir = None - - if cert: - if not isinstance(cert, basestring): - conn.cert_file = cert[0] - conn.key_file = cert[1] - else: - conn.cert_file = cert - conn.key_file = None - if conn.cert_file and not os.path.exists(conn.cert_file): - raise IOError("Could not find the TLS certificate file, " - "invalid path: {}".format(conn.cert_file)) - if conn.key_file and not os.path.exists(conn.key_file): - raise IOError("Could not find the TLS key file, " - "invalid path: {}".format(conn.key_file)) - - def build_response(self, req, resp): - """Builds a :class:`Response ` object from a urllib3 - response. This should not be called from user code, and is only exposed - for use when subclassing the - :class:`HTTPAdapter ` - - :param req: The :class:`PreparedRequest ` used to generate the response. - :param resp: The urllib3 response object. - :rtype: requests.Response - """ - response = Response() - - # Fallback to None if there's no status_code, for whatever reason. - response.status_code = getattr(resp, 'status', None) - - # Make headers case-insensitive. - response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {})) - - # Set encoding. - response.encoding = get_encoding_from_headers(response.headers) - response.raw = resp - response.reason = response.raw.reason - - if isinstance(req.url, bytes): - response.url = req.url.decode('utf-8') - else: - response.url = req.url - - # Add new cookies from the server. - extract_cookies_to_jar(response.cookies, req, resp) - - # Give the Response some context. - response.request = req - response.connection = self - - return response - - def get_connection(self, url, proxies=None): - """Returns a urllib3 connection for the given URL. This should not be - called from user code, and is only exposed for use when subclassing the - :class:`HTTPAdapter `. - - :param url: The URL to connect to. - :param proxies: (optional) A Requests-style dictionary of proxies used on this request. - :rtype: urllib3.ConnectionPool - """ - proxy = select_proxy(url, proxies) - - if proxy: - proxy = prepend_scheme_if_needed(proxy, 'http') - proxy_url = parse_url(proxy) - if not proxy_url.host: - raise InvalidProxyURL("Please check proxy URL. It is malformed" - " and could be missing the host.") - proxy_manager = self.proxy_manager_for(proxy) - conn = proxy_manager.connection_from_url(url) - else: - # Only scheme should be lower case - parsed = urlparse(url) - url = parsed.geturl() - conn = self.poolmanager.connection_from_url(url) - - return conn - - def close(self): - """Disposes of any internal state. - - Currently, this closes the PoolManager and any active ProxyManager, - which closes any pooled connections. - """ - self.poolmanager.clear() - for proxy in self.proxy_manager.values(): - proxy.clear() - - def request_url(self, request, proxies): - """Obtain the url to use when making the final request. - - If the message is being sent through a HTTP proxy, the full URL has to - be used. Otherwise, we should only use the path portion of the URL. - - This should not be called from user code, and is only exposed for use - when subclassing the - :class:`HTTPAdapter `. - - :param request: The :class:`PreparedRequest ` being sent. - :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs. - :rtype: str - """ - proxy = select_proxy(request.url, proxies) - scheme = urlparse(request.url).scheme - - is_proxied_http_request = (proxy and scheme != 'https') - using_socks_proxy = False - if proxy: - proxy_scheme = urlparse(proxy).scheme.lower() - using_socks_proxy = proxy_scheme.startswith('socks') - - url = request.path_url - if is_proxied_http_request and not using_socks_proxy: - url = urldefragauth(request.url) - - return url - - def add_headers(self, request, **kwargs): - """Add any headers needed by the connection. As of v2.0 this does - nothing by default, but is left for overriding by users that subclass - the :class:`HTTPAdapter `. - - This should not be called from user code, and is only exposed for use - when subclassing the - :class:`HTTPAdapter `. - - :param request: The :class:`PreparedRequest ` to add headers to. - :param kwargs: The keyword arguments from the call to send(). - """ - pass - - def proxy_headers(self, proxy): - """Returns a dictionary of the headers to add to any request sent - through a proxy. This works with urllib3 magic to ensure that they are - correctly sent to the proxy, rather than in a tunnelled request if - CONNECT is being used. - - This should not be called from user code, and is only exposed for use - when subclassing the - :class:`HTTPAdapter `. - - :param proxy: The url of the proxy being used for this request. - :rtype: dict - """ - headers = {} - username, password = get_auth_from_url(proxy) - - if username: - headers['Proxy-Authorization'] = _basic_auth_str(username, - password) - - return headers - - def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None): - """Sends PreparedRequest object. Returns Response object. - - :param request: The :class:`PreparedRequest ` being sent. - :param stream: (optional) Whether to stream the request content. - :param timeout: (optional) How long to wait for the server to send - data before giving up, as a float, or a :ref:`(connect timeout, - read timeout) ` tuple. - :type timeout: float or tuple or urllib3 Timeout object - :param verify: (optional) Either a boolean, in which case it controls whether - we verify the server's TLS certificate, or a string, in which case it - must be a path to a CA bundle to use - :param cert: (optional) Any user-provided SSL certificate to be trusted. - :param proxies: (optional) The proxies dictionary to apply to the request. - :rtype: requests.Response - """ - - try: - conn = self.get_connection(request.url, proxies) - except LocationValueError as e: - raise InvalidURL(e, request=request) - - self.cert_verify(conn, request.url, verify, cert) - url = self.request_url(request, proxies) - self.add_headers(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) - - chunked = not (request.body is None or 'Content-Length' in request.headers) - - if isinstance(timeout, tuple): - try: - connect, read = timeout - timeout = TimeoutSauce(connect=connect, read=read) - except ValueError as e: - # this may raise a string formatting error. - err = ("Invalid timeout {}. Pass a (connect, read) " - "timeout tuple, or a single float to set " - "both timeouts to the same value".format(timeout)) - raise ValueError(err) - elif isinstance(timeout, TimeoutSauce): - pass - else: - timeout = TimeoutSauce(connect=timeout, read=timeout) - - try: - if not chunked: - resp = conn.urlopen( - method=request.method, - url=url, - body=request.body, - headers=request.headers, - redirect=False, - assert_same_host=False, - preload_content=False, - decode_content=False, - retries=self.max_retries, - timeout=timeout - ) - - # Send the request. - else: - if hasattr(conn, 'proxy_pool'): - conn = conn.proxy_pool - - low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT) - - try: - low_conn.putrequest(request.method, - url, - skip_accept_encoding=True) - - for header, value in request.headers.items(): - low_conn.putheader(header, value) - - low_conn.endheaders() - - for i in request.body: - low_conn.send(hex(len(i))[2:].encode('utf-8')) - low_conn.send(b'\r\n') - low_conn.send(i) - low_conn.send(b'\r\n') - low_conn.send(b'0\r\n\r\n') - - # Receive the response from the server - try: - # For Python 2.7, use buffering of HTTP responses - r = low_conn.getresponse(buffering=True) - except TypeError: - # For compatibility with Python 3.3+ - r = low_conn.getresponse() - - resp = HTTPResponse.from_httplib( - r, - pool=conn, - connection=low_conn, - preload_content=False, - decode_content=False - ) - except: - # If we hit any problems here, clean up the connection. - # Then, reraise so that we can handle the actual exception. - low_conn.close() - raise - - except (ProtocolError, socket.error) as err: - raise ConnectionError(err, request=request) - - except MaxRetryError as e: - if isinstance(e.reason, ConnectTimeoutError): - # TODO: Remove this in 3.0.0: see #2811 - if not isinstance(e.reason, NewConnectionError): - raise ConnectTimeout(e, request=request) - - if isinstance(e.reason, ResponseError): - raise RetryError(e, request=request) - - if isinstance(e.reason, _ProxyError): - raise ProxyError(e, request=request) - - if isinstance(e.reason, _SSLError): - # This branch is for urllib3 v1.22 and later. - raise SSLError(e, request=request) - - raise ConnectionError(e, request=request) - - except ClosedPoolError as e: - raise ConnectionError(e, request=request) - - except _ProxyError as e: - raise ProxyError(e) - - except (_SSLError, _HTTPError) as e: - if isinstance(e, _SSLError): - # This branch is for urllib3 versions earlier than v1.22 - raise SSLError(e, request=request) - elif isinstance(e, ReadTimeoutError): - raise ReadTimeout(e, request=request) - else: - raise - - return self.build_response(request, resp) diff --git a/requests/compat.py b/requests/compat.py deleted file mode 100644 index c44b35ef..00000000 --- a/requests/compat.py +++ /dev/null @@ -1,70 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -requests.compat -~~~~~~~~~~~~~~~ - -This module handles import compatibility issues between Python 2 and -Python 3. -""" - -import chardet - -import sys - -# ------- -# Pythons -# ------- - -# Syntax sugar. -_ver = sys.version_info - -#: Python 2.x? -is_py2 = (_ver[0] == 2) - -#: Python 3.x? -is_py3 = (_ver[0] == 3) - -try: - import simplejson as json -except ImportError: - import json - -# --------- -# Specifics -# --------- - -if is_py2: - from urllib import ( - quote, unquote, quote_plus, unquote_plus, urlencode, getproxies, - proxy_bypass, proxy_bypass_environment, getproxies_environment) - from urlparse import urlparse, urlunparse, urljoin, urlsplit, urldefrag - from urllib2 import parse_http_list - import cookielib - from Cookie import Morsel - from StringIO import StringIO - from collections import Callable, Mapping, MutableMapping, OrderedDict - - - builtin_str = str - bytes = str - str = unicode - basestring = basestring - numeric_types = (int, long, float) - integer_types = (int, long) - -elif is_py3: - from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, quote_plus, unquote_plus, urldefrag - from urllib.request import parse_http_list, getproxies, proxy_bypass, proxy_bypass_environment, getproxies_environment - from http import cookiejar as cookielib - from http.cookies import Morsel - from io import StringIO - from collections import OrderedDict - from collections.abc import Callable, Mapping, MutableMapping - - builtin_str = str - str = str - bytes = bytes - basestring = (str, bytes) - numeric_types = (int, float) - integer_types = (int,) diff --git a/requests/help.py b/requests/help.py deleted file mode 100644 index e53d35ef..00000000 --- a/requests/help.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Module containing bug report helper(s).""" -from __future__ import print_function - -import json -import platform -import sys -import ssl - -import idna -import urllib3 -import chardet - -from . import __version__ as requests_version - -try: - from urllib3.contrib import pyopenssl -except ImportError: - pyopenssl = None - OpenSSL = None - cryptography = None -else: - import OpenSSL - import cryptography - - -def _implementation(): - """Return a dict with the Python implementation and version. - - Provide both the name and the version of the Python implementation - currently running. For example, on CPython 2.7.5 it will return - {'name': 'CPython', 'version': '2.7.5'}. - - This function works best on CPython and PyPy: in particular, it probably - doesn't work for Jython or IronPython. Future investigation should be done - to work out the correct shape of the code for those platforms. - """ - implementation = platform.python_implementation() - - if implementation == 'CPython': - implementation_version = platform.python_version() - elif implementation == 'PyPy': - implementation_version = '%s.%s.%s' % (sys.pypy_version_info.major, - sys.pypy_version_info.minor, - sys.pypy_version_info.micro) - if sys.pypy_version_info.releaselevel != 'final': - implementation_version = ''.join([ - implementation_version, sys.pypy_version_info.releaselevel - ]) - elif implementation == 'Jython': - implementation_version = platform.python_version() # Complete Guess - elif implementation == 'IronPython': - implementation_version = platform.python_version() # Complete Guess - else: - implementation_version = 'Unknown' - - return {'name': implementation, 'version': implementation_version} - - -def info(): - """Generate information for a bug report.""" - try: - platform_info = { - 'system': platform.system(), - 'release': platform.release(), - } - except IOError: - platform_info = { - 'system': 'Unknown', - 'release': 'Unknown', - } - - implementation_info = _implementation() - urllib3_info = {'version': urllib3.__version__} - chardet_info = {'version': chardet.__version__} - - pyopenssl_info = { - 'version': None, - 'openssl_version': '', - } - if OpenSSL: - pyopenssl_info = { - 'version': OpenSSL.__version__, - 'openssl_version': '%x' % OpenSSL.SSL.OPENSSL_VERSION_NUMBER, - } - cryptography_info = { - 'version': getattr(cryptography, '__version__', ''), - } - idna_info = { - 'version': getattr(idna, '__version__', ''), - } - - system_ssl = ssl.OPENSSL_VERSION_NUMBER - system_ssl_info = { - 'version': '%x' % system_ssl if system_ssl is not None else '' - } - - return { - 'platform': platform_info, - 'implementation': implementation_info, - 'system_ssl': system_ssl_info, - 'using_pyopenssl': pyopenssl is not None, - 'pyOpenSSL': pyopenssl_info, - 'urllib3': urllib3_info, - 'chardet': chardet_info, - 'cryptography': cryptography_info, - 'idna': idna_info, - 'requests': { - 'version': requests_version, - }, - } - - -def main(): - """Pretty-print the bug information as JSON.""" - print(json.dumps(info(), sort_keys=True, indent=2)) - - -if __name__ == '__main__': - main() diff --git a/requests/packages.py b/requests/packages.py deleted file mode 100644 index 7232fe0f..00000000 --- a/requests/packages.py +++ /dev/null @@ -1,14 +0,0 @@ -import sys - -# This code exists for backwards compatibility reasons. -# I don't like it either. Just look the other way. :) - -for package in ('urllib3', 'idna', 'chardet'): - locals()[package] = __import__(package) - # This traversal is apparently necessary such that the identities are - # preserved (requests.packages.urllib3.* is urllib3.*) - for mod in list(sys.modules): - if mod == package or mod.startswith(package + '.'): - sys.modules['requests.packages.' + mod] = sys.modules[mod] - -# Kinda cool, though, right? diff --git a/requests/sessions.py b/requests/sessions.py deleted file mode 100644 index d73d700f..00000000 --- a/requests/sessions.py +++ /dev/null @@ -1,770 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -requests.session -~~~~~~~~~~~~~~~~ - -This module provides a Session object to manage and persist settings across -requests (cookies, auth, proxies). -""" -import os -import sys -import time -from datetime import timedelta - -from .auth import _basic_auth_str -from .compat import cookielib, is_py3, OrderedDict, urljoin, urlparse, Mapping -from .cookies import ( - cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies) -from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT -from .hooks import default_hooks, dispatch_hook -from ._internal_utils import to_native_string -from .utils import to_key_val_list, default_headers, DEFAULT_PORTS -from .exceptions import ( - TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) - -from .structures import CaseInsensitiveDict -from .adapters import HTTPAdapter - -from .utils import ( - requote_uri, get_environ_proxies, get_netrc_auth, should_bypass_proxies, - get_auth_from_url, rewind_body -) - -from .status_codes import codes - -# formerly defined here, reexposed here for backward compatibility -from .models import REDIRECT_STATI - -# Preferred clock, based on which one is more accurate on a given system. -if sys.platform == 'win32': - try: # Python 3.4+ - preferred_clock = time.perf_counter - except AttributeError: # Earlier than Python 3. - preferred_clock = time.clock -else: - preferred_clock = time.time - - -def merge_setting(request_setting, session_setting, dict_class=OrderedDict): - """Determines appropriate setting for a given request, taking into account - the explicit setting on that request, and the setting in the session. If a - setting is a dictionary, they will be merged together using `dict_class` - """ - - if session_setting is None: - return request_setting - - if request_setting is None: - return session_setting - - # Bypass if not a dictionary (e.g. verify) - if not ( - isinstance(session_setting, Mapping) and - isinstance(request_setting, Mapping) - ): - return request_setting - - merged_setting = dict_class(to_key_val_list(session_setting)) - merged_setting.update(to_key_val_list(request_setting)) - - # Remove keys that are set to None. Extract keys first to avoid altering - # the dictionary during iteration. - none_keys = [k for (k, v) in merged_setting.items() if v is None] - for key in none_keys: - del merged_setting[key] - - return merged_setting - - -def merge_hooks(request_hooks, session_hooks, dict_class=OrderedDict): - """Properly merges both requests and session hooks. - - This is necessary because when request_hooks == {'response': []}, the - merge breaks Session hooks entirely. - """ - if session_hooks is None or session_hooks.get('response') == []: - return request_hooks - - if request_hooks is None or request_hooks.get('response') == []: - return session_hooks - - return merge_setting(request_hooks, session_hooks, dict_class) - - -class SessionRedirectMixin(object): - - def get_redirect_target(self, resp): - """Receives a Response. Returns a redirect URI or ``None``""" - # Due to the nature of how requests processes redirects this method will - # be called at least once upon the original response and at least twice - # on each subsequent redirect response (if any). - # If a custom mixin is used to handle this logic, it may be advantageous - # to cache the redirect location onto the response object as a private - # attribute. - if resp.is_redirect: - location = resp.headers['location'] - # Currently the underlying http module on py3 decode headers - # in latin1, but empirical evidence suggests that latin1 is very - # rarely used with non-ASCII characters in HTTP headers. - # It is more likely to get UTF8 header rather than latin1. - # This causes incorrect handling of UTF8 encoded location headers. - # To solve this, we re-encode the location in latin1. - if is_py3: - location = location.encode('latin1') - return to_native_string(location, 'utf8') - return None - - def should_strip_auth(self, old_url, new_url): - """Decide whether Authorization header should be removed when redirecting""" - old_parsed = urlparse(old_url) - new_parsed = urlparse(new_url) - if old_parsed.hostname != new_parsed.hostname: - return True - # Special case: allow http -> https redirect when using the standard - # ports. This isn't specified by RFC 7235, but is kept to avoid - # breaking backwards compatibility with older versions of requests - # that allowed any redirects on the same host. - if (old_parsed.scheme == 'http' and old_parsed.port in (80, None) - and new_parsed.scheme == 'https' and new_parsed.port in (443, None)): - return False - - # Handle default port usage corresponding to scheme. - changed_port = old_parsed.port != new_parsed.port - changed_scheme = old_parsed.scheme != new_parsed.scheme - default_port = (DEFAULT_PORTS.get(old_parsed.scheme, None), None) - if (not changed_scheme and old_parsed.port in default_port - and new_parsed.port in default_port): - return False - - # Standard case: root URI must match - return changed_port or changed_scheme - - def resolve_redirects(self, resp, req, stream=False, timeout=None, - verify=True, cert=None, proxies=None, yield_requests=False, **adapter_kwargs): - """Receives a Response. Returns a generator of Responses or Requests.""" - - hist = [] # keep track of history - - url = self.get_redirect_target(resp) - previous_fragment = urlparse(req.url).fragment - while url: - prepared_request = req.copy() - - # Update history and keep track of redirects. - # resp.history must ignore the original request in this loop - hist.append(resp) - resp.history = hist[1:] - - try: - resp.content # Consume socket so it can be released - except (ChunkedEncodingError, ContentDecodingError, RuntimeError): - resp.raw.read(decode_content=False) - - if len(resp.history) >= self.max_redirects: - raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects, response=resp) - - # Release the connection back into the pool. - resp.close() - - # Handle redirection without scheme (see: RFC 1808 Section 4) - if url.startswith('//'): - parsed_rurl = urlparse(resp.url) - url = '%s:%s' % (to_native_string(parsed_rurl.scheme), url) - - # Normalize url case and attach previous fragment if needed (RFC 7231 7.1.2) - parsed = urlparse(url) - if parsed.fragment == '' and previous_fragment: - parsed = parsed._replace(fragment=previous_fragment) - elif parsed.fragment: - previous_fragment = parsed.fragment - url = parsed.geturl() - - # Facilitate relative 'location' headers, as allowed by RFC 7231. - # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') - # Compliant with RFC3986, we percent encode the url. - if not parsed.netloc: - url = urljoin(resp.url, requote_uri(url)) - else: - url = requote_uri(url) - - prepared_request.url = to_native_string(url) - - self.rebuild_method(prepared_request, resp) - - # https://github.com/requests/requests/issues/1084 - if resp.status_code not in (codes.temporary_redirect, codes.permanent_redirect): - # https://github.com/requests/requests/issues/3490 - purged_headers = ('Content-Length', 'Content-Type', 'Transfer-Encoding') - for header in purged_headers: - prepared_request.headers.pop(header, None) - prepared_request.body = None - - headers = prepared_request.headers - try: - del headers['Cookie'] - except KeyError: - pass - - # Extract any cookies sent on the response to the cookiejar - # in the new request. Because we've mutated our copied prepared - # request, use the old one that we haven't yet touched. - extract_cookies_to_jar(prepared_request._cookies, req, resp.raw) - merge_cookies(prepared_request._cookies, self.cookies) - prepared_request.prepare_cookies(prepared_request._cookies) - - # Rebuild auth and proxy information. - proxies = self.rebuild_proxies(prepared_request, proxies) - self.rebuild_auth(prepared_request, resp) - - # A failed tell() sets `_body_position` to `object()`. This non-None - # value ensures `rewindable` will be True, allowing us to raise an - # UnrewindableBodyError, instead of hanging the connection. - rewindable = ( - prepared_request._body_position is not None and - ('Content-Length' in headers or 'Transfer-Encoding' in headers) - ) - - # Attempt to rewind consumed file-like object. - if rewindable: - rewind_body(prepared_request) - - # Override the original request. - req = prepared_request - - if yield_requests: - yield req - else: - - resp = self.send( - req, - stream=stream, - timeout=timeout, - verify=verify, - cert=cert, - proxies=proxies, - allow_redirects=False, - **adapter_kwargs - ) - - extract_cookies_to_jar(self.cookies, prepared_request, resp.raw) - - # extract redirect url, if any, for the next loop - url = self.get_redirect_target(resp) - yield resp - - def rebuild_auth(self, prepared_request, response): - """When being redirected we may want to strip authentication from the - request to avoid leaking credentials. This method intelligently removes - and reapplies authentication where possible to avoid credential loss. - """ - headers = prepared_request.headers - url = prepared_request.url - - if 'Authorization' in headers and self.should_strip_auth(response.request.url, url): - # If we get redirected to a new host, we should strip out any - # authentication headers. - del headers['Authorization'] - - # .netrc might have more auth for us on our new host. - new_auth = get_netrc_auth(url) if self.trust_env else None - if new_auth is not None: - prepared_request.prepare_auth(new_auth) - - return - - def rebuild_proxies(self, prepared_request, proxies): - """This method re-evaluates the proxy configuration by considering the - environment variables. If we are redirected to a URL covered by - NO_PROXY, we strip the proxy configuration. Otherwise, we set missing - proxy keys for this URL (in case they were stripped by a previous - redirect). - - This method also replaces the Proxy-Authorization header where - necessary. - - :rtype: dict - """ - proxies = proxies if proxies is not None else {} - headers = prepared_request.headers - url = prepared_request.url - scheme = urlparse(url).scheme - new_proxies = proxies.copy() - no_proxy = proxies.get('no_proxy') - - bypass_proxy = should_bypass_proxies(url, no_proxy=no_proxy) - if self.trust_env and not bypass_proxy: - environ_proxies = get_environ_proxies(url, no_proxy=no_proxy) - - proxy = environ_proxies.get(scheme, environ_proxies.get('all')) - - if proxy: - new_proxies.setdefault(scheme, proxy) - - if 'Proxy-Authorization' in headers: - del headers['Proxy-Authorization'] - - try: - username, password = get_auth_from_url(new_proxies[scheme]) - except KeyError: - username, password = None, None - - if username and password: - headers['Proxy-Authorization'] = _basic_auth_str(username, password) - - return new_proxies - - def rebuild_method(self, prepared_request, response): - """When being redirected we may want to change the method of the request - based on certain specs or browser behavior. - """ - method = prepared_request.method - - # https://tools.ietf.org/html/rfc7231#section-6.4.4 - if response.status_code == codes.see_other and method != 'HEAD': - method = 'GET' - - # Do what the browsers do, despite standards... - # First, turn 302s into GETs. - if response.status_code == codes.found and method != 'HEAD': - method = 'GET' - - # Second, if a POST is responded to with a 301, turn it into a GET. - # This bizarre behaviour is explained in Issue 1704. - if response.status_code == codes.moved and method == 'POST': - method = 'GET' - - prepared_request.method = method - - -class Session(SessionRedirectMixin): - """A Requests session. - - Provides cookie persistence, connection-pooling, and configuration. - - Basic Usage:: - - >>> import requests - >>> s = requests.Session() - >>> s.get('https://httpbin.org/get') - - - Or as a context manager:: - - >>> with requests.Session() as s: - >>> s.get('https://httpbin.org/get') - - """ - - __attrs__ = [ - 'headers', 'cookies', 'auth', 'proxies', 'hooks', 'params', 'verify', - 'cert', 'prefetch', 'adapters', 'stream', 'trust_env', - 'max_redirects', - ] - - def __init__(self): - - #: A case-insensitive dictionary of headers to be sent on each - #: :class:`Request ` sent from this - #: :class:`Session `. - self.headers = default_headers() - - #: Default Authentication tuple or object to attach to - #: :class:`Request `. - self.auth = None - - #: Dictionary mapping protocol or protocol and host to the URL of the proxy - #: (e.g. {'http': 'foo.bar:3128', 'http://host.name': 'foo.bar:4012'}) to - #: be used on each :class:`Request `. - self.proxies = {} - - #: Event-handling hooks. - self.hooks = default_hooks() - - #: Dictionary of querystring data to attach to each - #: :class:`Request `. The dictionary values may be lists for - #: representing multivalued query parameters. - self.params = {} - - #: Stream response content default. - self.stream = False - - #: SSL Verification default. - self.verify = True - - #: SSL client certificate default, if String, path to ssl client - #: cert file (.pem). If Tuple, ('cert', 'key') pair. - self.cert = None - - #: Maximum number of redirects allowed. If the request exceeds this - #: limit, a :class:`TooManyRedirects` exception is raised. - #: This defaults to requests.models.DEFAULT_REDIRECT_LIMIT, which is - #: 30. - self.max_redirects = DEFAULT_REDIRECT_LIMIT - - #: Trust environment settings for proxy configuration, default - #: authentication and similar. - self.trust_env = True - - #: A CookieJar containing all currently outstanding cookies set on this - #: session. By default it is a - #: :class:`RequestsCookieJar `, but - #: may be any other ``cookielib.CookieJar`` compatible object. - self.cookies = cookiejar_from_dict({}) - - # Default connection adapters. - self.adapters = OrderedDict() - self.mount('https://', HTTPAdapter()) - self.mount('http://', HTTPAdapter()) - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - def prepare_request(self, request): - """Constructs a :class:`PreparedRequest ` for - transmission and returns it. The :class:`PreparedRequest` has settings - merged from the :class:`Request ` instance and those of the - :class:`Session`. - - :param request: :class:`Request` instance to prepare with this - session's settings. - :rtype: requests.PreparedRequest - """ - cookies = request.cookies or {} - - # Bootstrap CookieJar. - if not isinstance(cookies, cookielib.CookieJar): - cookies = cookiejar_from_dict(cookies) - - # Merge with session cookies - merged_cookies = merge_cookies( - merge_cookies(RequestsCookieJar(), self.cookies), cookies) - - # Set environment's basic authentication if not explicitly set. - auth = request.auth - if self.trust_env and not auth and not self.auth: - auth = get_netrc_auth(request.url) - - p = PreparedRequest() - p.prepare( - method=request.method.upper(), - url=request.url, - files=request.files, - data=request.data, - json=request.json, - headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict), - params=merge_setting(request.params, self.params), - auth=merge_setting(auth, self.auth), - cookies=merged_cookies, - hooks=merge_hooks(request.hooks, self.hooks), - ) - return p - - def request(self, method, url, - params=None, data=None, headers=None, cookies=None, files=None, - auth=None, timeout=None, allow_redirects=True, proxies=None, - hooks=None, stream=None, verify=None, cert=None, json=None): - """Constructs a :class:`Request `, prepares it and sends it. - Returns :class:`Response ` object. - - :param method: method for the new :class:`Request` object. - :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query - string for the :class:`Request`. - :param data: (optional) Dictionary, list of tuples, bytes, or file-like - object to send in the body of the :class:`Request`. - :param json: (optional) json to send in the body of the - :class:`Request`. - :param headers: (optional) Dictionary of HTTP Headers to send with the - :class:`Request`. - :param cookies: (optional) Dict or CookieJar object to send with the - :class:`Request`. - :param files: (optional) Dictionary of ``'filename': file-like-objects`` - for multipart encoding upload. - :param auth: (optional) Auth tuple or callable to enable - Basic/Digest/Custom HTTP Auth. - :param timeout: (optional) How long to wait for the server to send - data before giving up, as a float, or a :ref:`(connect timeout, - read timeout) ` tuple. - :type timeout: float or tuple - :param allow_redirects: (optional) Set to True by default. - :type allow_redirects: bool - :param proxies: (optional) Dictionary mapping protocol or protocol and - hostname to the URL of the proxy. - :param stream: (optional) whether to immediately download the response - content. Defaults to ``False``. - :param verify: (optional) Either a boolean, in which case it controls whether we verify - the server's TLS certificate, or a string, in which case it must be a path - to a CA bundle to use. Defaults to ``True``. - :param cert: (optional) if String, path to ssl client cert file (.pem). - If Tuple, ('cert', 'key') pair. - :rtype: requests.Response - """ - # Create the Request. - req = Request( - method=method.upper(), - url=url, - headers=headers, - files=files, - data=data or {}, - json=json, - params=params or {}, - auth=auth, - cookies=cookies, - hooks=hooks, - ) - prep = self.prepare_request(req) - - proxies = proxies or {} - - settings = self.merge_environment_settings( - prep.url, proxies, stream, verify, cert - ) - - # Send the request. - send_kwargs = { - 'timeout': timeout, - 'allow_redirects': allow_redirects, - } - send_kwargs.update(settings) - resp = self.send(prep, **send_kwargs) - - return resp - - def get(self, url, **kwargs): - r"""Sends a GET request. Returns :class:`Response` object. - - :param url: URL for the new :class:`Request` object. - :param \*\*kwargs: Optional arguments that ``request`` takes. - :rtype: requests.Response - """ - - kwargs.setdefault('allow_redirects', True) - return self.request('GET', url, **kwargs) - - def options(self, url, **kwargs): - r"""Sends a OPTIONS request. Returns :class:`Response` object. - - :param url: URL for the new :class:`Request` object. - :param \*\*kwargs: Optional arguments that ``request`` takes. - :rtype: requests.Response - """ - - kwargs.setdefault('allow_redirects', True) - return self.request('OPTIONS', url, **kwargs) - - def head(self, url, **kwargs): - r"""Sends a HEAD request. Returns :class:`Response` object. - - :param url: URL for the new :class:`Request` object. - :param \*\*kwargs: Optional arguments that ``request`` takes. - :rtype: requests.Response - """ - - kwargs.setdefault('allow_redirects', False) - return self.request('HEAD', url, **kwargs) - - def post(self, url, data=None, json=None, **kwargs): - r"""Sends a POST request. Returns :class:`Response` object. - - :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, list of tuples, bytes, or file-like - object to send in the body of the :class:`Request`. - :param json: (optional) json to send in the body of the :class:`Request`. - :param \*\*kwargs: Optional arguments that ``request`` takes. - :rtype: requests.Response - """ - - return self.request('POST', url, data=data, json=json, **kwargs) - - def put(self, url, data=None, **kwargs): - r"""Sends a PUT request. Returns :class:`Response` object. - - :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, list of tuples, bytes, or file-like - object to send in the body of the :class:`Request`. - :param \*\*kwargs: Optional arguments that ``request`` takes. - :rtype: requests.Response - """ - - return self.request('PUT', url, data=data, **kwargs) - - def patch(self, url, data=None, **kwargs): - r"""Sends a PATCH request. Returns :class:`Response` object. - - :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, list of tuples, bytes, or file-like - object to send in the body of the :class:`Request`. - :param \*\*kwargs: Optional arguments that ``request`` takes. - :rtype: requests.Response - """ - - return self.request('PATCH', url, data=data, **kwargs) - - def delete(self, url, **kwargs): - r"""Sends a DELETE request. Returns :class:`Response` object. - - :param url: URL for the new :class:`Request` object. - :param \*\*kwargs: Optional arguments that ``request`` takes. - :rtype: requests.Response - """ - - return self.request('DELETE', url, **kwargs) - - def send(self, request, **kwargs): - """Send a given PreparedRequest. - - :rtype: requests.Response - """ - # Set defaults that the hooks can utilize to ensure they always have - # the correct parameters to reproduce the previous request. - kwargs.setdefault('stream', self.stream) - kwargs.setdefault('verify', self.verify) - kwargs.setdefault('cert', self.cert) - kwargs.setdefault('proxies', self.proxies) - - # It's possible that users might accidentally send a Request object. - # Guard against that specific failure case. - if isinstance(request, Request): - raise ValueError('You can only send PreparedRequests.') - - # Set up variables needed for resolve_redirects and dispatching of hooks - allow_redirects = kwargs.pop('allow_redirects', True) - stream = kwargs.get('stream') - hooks = request.hooks - - # Get the appropriate adapter to use - adapter = self.get_adapter(url=request.url) - - # Start time (approximately) of the request - start = preferred_clock() - - # Send the request - r = adapter.send(request, **kwargs) - - # Total elapsed time of the request (approximately) - elapsed = preferred_clock() - start - r.elapsed = timedelta(seconds=elapsed) - - # Response manipulation hooks - r = dispatch_hook('response', hooks, r, **kwargs) - - # Persist cookies - if r.history: - - # If the hooks create history then we want those cookies too - for resp in r.history: - extract_cookies_to_jar(self.cookies, resp.request, resp.raw) - - extract_cookies_to_jar(self.cookies, request, r.raw) - - # Redirect resolving generator. - gen = self.resolve_redirects(r, request, **kwargs) - - # Resolve redirects if allowed. - history = [resp for resp in gen] if allow_redirects else [] - - # Shuffle things around if there's history. - if history: - # Insert the first (original) request at the start - history.insert(0, r) - # Get the last request made - r = history.pop() - r.history = history - - # If redirects aren't being followed, store the response on the Request for Response.next(). - if not allow_redirects: - try: - r._next = next(self.resolve_redirects(r, request, yield_requests=True, **kwargs)) - except StopIteration: - pass - - if not stream: - r.content - - return r - - def merge_environment_settings(self, url, proxies, stream, verify, cert): - """ - Check the environment and merge it with some settings. - - :rtype: dict - """ - # Gather clues from the surrounding environment. - if self.trust_env: - # Set environment's proxies. - no_proxy = proxies.get('no_proxy') if proxies is not None else None - env_proxies = get_environ_proxies(url, no_proxy=no_proxy) - for (k, v) in env_proxies.items(): - proxies.setdefault(k, v) - - # Look for requests environment configuration and be compatible - # with cURL. - if verify is True or verify is None: - verify = (os.environ.get('REQUESTS_CA_BUNDLE') or - os.environ.get('CURL_CA_BUNDLE')) - - # Merge all the kwargs. - proxies = merge_setting(proxies, self.proxies) - stream = merge_setting(stream, self.stream) - verify = merge_setting(verify, self.verify) - cert = merge_setting(cert, self.cert) - - return {'verify': verify, 'proxies': proxies, 'stream': stream, - 'cert': cert} - - def get_adapter(self, url): - """ - Returns the appropriate connection adapter for the given URL. - - :rtype: requests.adapters.BaseAdapter - """ - for (prefix, adapter) in self.adapters.items(): - - if url.lower().startswith(prefix.lower()): - return adapter - - # Nothing matches :-/ - raise InvalidSchema("No connection adapters were found for '%s'" % url) - - def close(self): - """Closes all adapters and as such the session""" - for v in self.adapters.values(): - v.close() - - def mount(self, prefix, adapter): - """Registers a connection adapter to a prefix. - - Adapters are sorted in descending order by prefix length. - """ - self.adapters[prefix] = adapter - keys_to_move = [k for k in self.adapters if len(k) < len(prefix)] - - for key in keys_to_move: - self.adapters[key] = self.adapters.pop(key) - - def __getstate__(self): - state = {attr: getattr(self, attr, None) for attr in self.__attrs__} - return state - - def __setstate__(self, state): - for attr, value in state.items(): - setattr(self, attr, value) - - -def session(): - """ - Returns a :class:`Session` for context-management. - - .. deprecated:: 1.0.0 - - This method has been deprecated since version 1.0.0 and is only kept for - backwards compatibility. New code should use :class:`~requests.sessions.Session` - to create a session. This may be removed at a future date. - - :rtype: Session - """ - return Session() diff --git a/requests/status_codes.py b/requests/status_codes.py deleted file mode 100644 index 813e8c4e..00000000 --- a/requests/status_codes.py +++ /dev/null @@ -1,120 +0,0 @@ -# -*- coding: utf-8 -*- - -r""" -The ``codes`` object defines a mapping from common names for HTTP statuses -to their numerical codes, accessible either as attributes or as dictionary -items. - ->>> requests.codes['temporary_redirect'] -307 ->>> requests.codes.teapot -418 ->>> requests.codes['\o/'] -200 - -Some codes have multiple names, and both upper- and lower-case versions of -the names are allowed. For example, ``codes.ok``, ``codes.OK``, and -``codes.okay`` all correspond to the HTTP status code 200. -""" - -from .structures import LookupDict - -_codes = { - - # Informational. - 100: ('continue',), - 101: ('switching_protocols',), - 102: ('processing',), - 103: ('checkpoint',), - 122: ('uri_too_long', 'request_uri_too_long'), - 200: ('ok', 'okay', 'all_ok', 'all_okay', 'all_good', '\\o/', '✓'), - 201: ('created',), - 202: ('accepted',), - 203: ('non_authoritative_info', 'non_authoritative_information'), - 204: ('no_content',), - 205: ('reset_content', 'reset'), - 206: ('partial_content', 'partial'), - 207: ('multi_status', 'multiple_status', 'multi_stati', 'multiple_stati'), - 208: ('already_reported',), - 226: ('im_used',), - - # Redirection. - 300: ('multiple_choices',), - 301: ('moved_permanently', 'moved', '\\o-'), - 302: ('found',), - 303: ('see_other', 'other'), - 304: ('not_modified',), - 305: ('use_proxy',), - 306: ('switch_proxy',), - 307: ('temporary_redirect', 'temporary_moved', 'temporary'), - 308: ('permanent_redirect', - 'resume_incomplete', 'resume',), # These 2 to be removed in 3.0 - - # Client Error. - 400: ('bad_request', 'bad'), - 401: ('unauthorized',), - 402: ('payment_required', 'payment'), - 403: ('forbidden',), - 404: ('not_found', '-o-'), - 405: ('method_not_allowed', 'not_allowed'), - 406: ('not_acceptable',), - 407: ('proxy_authentication_required', 'proxy_auth', 'proxy_authentication'), - 408: ('request_timeout', 'timeout'), - 409: ('conflict',), - 410: ('gone',), - 411: ('length_required',), - 412: ('precondition_failed', 'precondition'), - 413: ('request_entity_too_large',), - 414: ('request_uri_too_large',), - 415: ('unsupported_media_type', 'unsupported_media', 'media_type'), - 416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'), - 417: ('expectation_failed',), - 418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'), - 421: ('misdirected_request',), - 422: ('unprocessable_entity', 'unprocessable'), - 423: ('locked',), - 424: ('failed_dependency', 'dependency'), - 425: ('unordered_collection', 'unordered'), - 426: ('upgrade_required', 'upgrade'), - 428: ('precondition_required', 'precondition'), - 429: ('too_many_requests', 'too_many'), - 431: ('header_fields_too_large', 'fields_too_large'), - 444: ('no_response', 'none'), - 449: ('retry_with', 'retry'), - 450: ('blocked_by_windows_parental_controls', 'parental_controls'), - 451: ('unavailable_for_legal_reasons', 'legal_reasons'), - 499: ('client_closed_request',), - - # Server Error. - 500: ('internal_server_error', 'server_error', '/o\\', '✗'), - 501: ('not_implemented',), - 502: ('bad_gateway',), - 503: ('service_unavailable', 'unavailable'), - 504: ('gateway_timeout',), - 505: ('http_version_not_supported', 'http_version'), - 506: ('variant_also_negotiates',), - 507: ('insufficient_storage',), - 509: ('bandwidth_limit_exceeded', 'bandwidth'), - 510: ('not_extended',), - 511: ('network_authentication_required', 'network_auth', 'network_authentication'), -} - -codes = LookupDict(name='status_codes') - -def _init(): - for code, titles in _codes.items(): - for title in titles: - setattr(codes, title, code) - if not title.startswith(('\\', '/')): - setattr(codes, title.upper(), code) - - def doc(code): - names = ', '.join('``%s``' % n for n in _codes[code]) - return '* %d: %s' % (code, names) - - global __doc__ - __doc__ = (__doc__ + '\n' + - '\n'.join(doc(code) for code in sorted(_codes)) - if __doc__ is not None else None) - -_init() diff --git a/requests/structures.py b/requests/structures.py deleted file mode 100644 index da930e28..00000000 --- a/requests/structures.py +++ /dev/null @@ -1,103 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -requests.structures -~~~~~~~~~~~~~~~~~~~ - -Data structures that power Requests. -""" - -from .compat import OrderedDict, Mapping, MutableMapping - - -class CaseInsensitiveDict(MutableMapping): - """A case-insensitive ``dict``-like object. - - Implements all methods and operations of - ``MutableMapping`` as well as dict's ``copy``. Also - provides ``lower_items``. - - All keys are expected to be strings. The structure remembers the - case of the last key to be set, and ``iter(instance)``, - ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()`` - will contain case-sensitive keys. However, querying and contains - testing is case insensitive:: - - cid = CaseInsensitiveDict() - cid['Accept'] = 'application/json' - cid['aCCEPT'] == 'application/json' # True - list(cid) == ['Accept'] # True - - For example, ``headers['content-encoding']`` will return the - value of a ``'Content-Encoding'`` response header, regardless - of how the header name was originally stored. - - If the constructor, ``.update``, or equality comparison - operations are given keys that have equal ``.lower()``s, the - behavior is undefined. - """ - - def __init__(self, data=None, **kwargs): - self._store = OrderedDict() - if data is None: - data = {} - self.update(data, **kwargs) - - def __setitem__(self, key, value): - # Use the lowercased key for lookups, but store the actual - # key alongside the value. - self._store[key.lower()] = (key, value) - - def __getitem__(self, key): - return self._store[key.lower()][1] - - def __delitem__(self, key): - del self._store[key.lower()] - - def __iter__(self): - return (casedkey for casedkey, mappedvalue in self._store.values()) - - def __len__(self): - return len(self._store) - - def lower_items(self): - """Like iteritems(), but with all lowercase keys.""" - return ( - (lowerkey, keyval[1]) - for (lowerkey, keyval) - in self._store.items() - ) - - def __eq__(self, other): - if isinstance(other, Mapping): - other = CaseInsensitiveDict(other) - else: - return NotImplemented - # Compare insensitively - return dict(self.lower_items()) == dict(other.lower_items()) - - # Copy is required - def copy(self): - return CaseInsensitiveDict(self._store.values()) - - def __repr__(self): - return str(dict(self.items())) - - -class LookupDict(dict): - """Dictionary lookup object.""" - - def __init__(self, name=None): - self.name = name - super(LookupDict, self).__init__() - - def __repr__(self): - return '' % (self.name) - - def __getitem__(self, key): - # We allow fall-through here, so values default to None - - return self.__dict__.get(key, None) - - def get(self, key, default=None): - return self.__dict__.get(key, default) diff --git a/requests/__init__.py b/requests3/__init__.py similarity index 67% rename from requests/__init__.py rename to requests3/__init__.py index bc168ee5..0ad0a281 100644 --- a/requests/__init__.py +++ b/requests3/__init__.py @@ -1,10 +1,8 @@ # -*- coding: utf-8 -*- - # __ # /__) _ _ _ _ _/ _ # / ( (- (/ (/ (- _) / _) # / - """ Requests HTTP Library ~~~~~~~~~~~~~~~~~~~~~ @@ -46,14 +44,12 @@ import warnings from .exceptions import RequestsDependencyWarning -def check_compatibility(urllib3_version, chardet_version): - urllib3_version = urllib3_version.split('.') - assert urllib3_version != ['dev'] # Verify urllib3 isn't installed from git. - +def check_compatibility(urllib3_version: str, chardet_version: str) -> None: + urllib3_version = urllib3_version.split(".") # type: ignore + assert urllib3_version != ["dev"] # Verify urllib3 isn't installed from git. # Sometimes, urllib3 only reports its version as 16.1. if len(urllib3_version) == 2: - urllib3_version.append('0') - + urllib3_version.append("0") # type: ignore # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) @@ -63,62 +59,75 @@ def check_compatibility(urllib3_version, chardet_version): assert minor <= 24 # Check chardet for compatibility. - major, minor, patch = chardet_version.split('.')[:3] - major, minor, patch = int(major), int(minor), int(patch) + major, minor, patch = chardet_version.split(".")[:3] + major, minor, patch = int(major), int(minor), int(patch) # type: ignore # chardet >= 3.0.2, < 3.1.0 - assert major == 3 - assert minor < 1 - assert patch >= 2 + assert major == 3 # type: ignore + assert minor < 1 # type: ignore + assert patch >= 2 # type: ignore -def _check_cryptography(cryptography_version): +def _check_cryptography(cryptography_version: str) -> None: # cryptography < 1.3.4 try: - cryptography_version = list(map(int, cryptography_version.split('.'))) + cryptography_version = list( + map(int, cryptography_version.split(".")) + ) # type: ignore except ValueError: return if cryptography_version < [1, 3, 4]: - warning = 'Old version of cryptography ({}) may cause slowdown.'.format(cryptography_version) + warning = "Old version of cryptography ({}) may cause slowdown.".format( + cryptography_version + ) warnings.warn(warning, RequestsDependencyWarning) + # Check imported dependencies for compatibility. try: check_compatibility(urllib3.__version__, chardet.__version__) except (AssertionError, ValueError): - warnings.warn("urllib3 ({}) or chardet ({}) doesn't match a supported " - "version!".format(urllib3.__version__, chardet.__version__), - RequestsDependencyWarning) + warnings.warn( + "urllib3 ({}) or chardet ({}) doesn't match a supported " + "version!".format(urllib3.__version__, chardet.__version__), + RequestsDependencyWarning, + ) # Attempt to enable urllib3's SNI support, if possible try: from urllib3.contrib import pyopenssl - pyopenssl.inject_into_urllib3() + pyopenssl.inject_into_urllib3() # Check cryptography version from cryptography import __version__ as cryptography_version + _check_cryptography(cryptography_version) except ImportError: pass - # urllib3's DependencyWarnings should be silenced. from urllib3.exceptions import DependencyWarning -warnings.simplefilter('ignore', DependencyWarning) + +warnings.simplefilter("ignore", DependencyWarning) from .__version__ import __title__, __description__, __url__, __version__ from .__version__ import __build__, __author__, __author_email__, __license__ from .__version__ import __copyright__, __cake__ from . import utils -from . import packages from .models import Request, Response, PreparedRequest from .api import request, get, head, post, patch, put, delete, options -from .sessions import session, Session +from .sessions import Session, AsyncSession from .status_codes import codes from .exceptions import ( - RequestException, Timeout, URLRequired, - TooManyRedirects, HTTPError, ConnectionError, - FileModeWarning, ConnectTimeout, ReadTimeout + RequestException, + Timeout, + URLRequired, + TooManyRedirects, + HTTPError, + ConnectionError, + FileModeWarning, + ConnectTimeout, + ReadTimeout, ) # Set default logging handler to avoid "No handler found" warnings. @@ -126,6 +135,5 @@ import logging from logging import NullHandler logging.getLogger(__name__).addHandler(NullHandler()) - # FileModeWarnings go off per the default. -warnings.simplefilter('default', FileModeWarning, append=True) +warnings.simplefilter("default", FileModeWarning, append=True) diff --git a/requests3/__version__.py b/requests3/__version__.py new file mode 100644 index 00000000..e7b3e552 --- /dev/null +++ b/requests3/__version__.py @@ -0,0 +1,13 @@ +# .-. .-. .-. . . .-. .-. .-. .-. +# |( |- |.| | | |- `-. | `-. +# ' ' `-' `-`.`-' `-' `-' ' `-' +__title__ = "requests" +__description__ = "Python HTTP for Humans." +__url__ = "http://python-requests.org" +__version__ = "3.0.0" +__build__ = 0x030000 +__author__ = "Kenneth Reitz" +__author_email__ = "me@kennethreitz.org" +__license__ = "Apache 2.0" +__copyright__ = "Copyright 2018 Kenneth Reitz" +__cake__ = u"\u2728 \U0001f370 \u2728" diff --git a/requests/_internal_utils.py b/requests3/_internal_utils.py similarity index 81% rename from requests/_internal_utils.py rename to requests3/_internal_utils.py index 759d9a56..a15e1116 100644 --- a/requests/_internal_utils.py +++ b/requests3/_internal_utils.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests._internal_utils ~~~~~~~~~~~~~~ @@ -8,7 +7,7 @@ Provides utility functions that are consumed internally by Requests which depend on extremely few external helpers (such as compat) """ -from .compat import is_py2, builtin_str, str +from .basics import builtin_str, str def to_native_string(string, encoding='ascii'): @@ -19,11 +18,7 @@ def to_native_string(string, encoding='ascii'): if isinstance(string, builtin_str): out = string else: - if is_py2: - out = string.encode(encoding) - else: - out = string.decode(encoding) - + out = string.decode(encoding) return out @@ -34,9 +29,12 @@ def unicode_is_ascii(u_string): and not Python 2 `str`. :rtype: bool """ - assert isinstance(u_string, str) + if not isinstance(u_string, str): + return None + try: u_string.encode('ascii') return True + except UnicodeEncodeError: return False diff --git a/requests3/adapters.py b/requests3/adapters.py new file mode 100644 index 00000000..ad183474 --- /dev/null +++ b/requests3/adapters.py @@ -0,0 +1,818 @@ +# -*- coding: utf-8 -*- +""" +requests.adapters +~~~~~~~~~~~~~~~~~ + +This module contains the transport adapters that Requests uses to define +and maintain connections. +""" + +import os.path +import socket + +from . import core +from .core._http._backends import TrioBackend +from .core._http.poolmanager import PoolManager, proxy_from_url +from .core._http._async.poolmanager import PoolManager as AsyncPoolManager +from .core._http.response import HTTPResponse +from .core._http.util import Timeout as TimeoutSauce +from .core._http.util.retry import Retry +from .core._http.exceptions import ClosedPoolError +from .core._http.exceptions import ConnectTimeoutError +from .core._http.exceptions import HTTPError as _HTTPError +from .core._http.exceptions import MaxRetryError +from .core._http.exceptions import NewConnectionError +from .core._http.exceptions import ProxyError as _ProxyError +from .core._http.exceptions import ProtocolError +from .core._http.exceptions import ReadTimeoutError +from .core._http.exceptions import SSLError as _SSLError +from .core._http.exceptions import ResponseError + +from .models import Response, AsyncResponse +from .basics import urlparse, basestring +from .utils import ( + DEFAULT_CA_BUNDLE_PATH, + get_encoding_from_headers, + prepend_scheme_if_needed, + get_auth_from_url, + urldefragauth, + select_proxy, +) +from .structures import HTTPHeaderDict +from .cookies import extract_cookies_to_jar +from .exceptions import ( + ConnectionError, + ConnectTimeout, + ReadTimeout, + SSLError, + ProxyError, + RetryError, + InvalidScheme, +) +from .auth import _basic_auth_str + +try: + from .core._http.contrib.socks import SOCKSProxyManager +except ImportError: + + def SOCKSProxyManager(*args, **kwargs): + raise InvalidScheme("Missing dependencies for SOCKS support.") + + +DEFAULT_POOLBLOCK = False +DEFAULT_POOLSIZE = 10 +DEFAULT_RETRIES = 0 +DEFAULT_POOL_TIMEOUT = None + + +def _pool_kwargs(verify, cert): + """Create a dictionary of keyword arguments to pass to a + :class:`PoolManager ` with the + necessary SSL configuration. + + :param verify: Whether we should actually verify the certificate; + optionally a path to a CA certificate bundle or + directory of CA certificates. + :param cert: The path to the client certificate and key, if any. + This can either be the path to the certificate and + key concatenated in a single file, or as a tuple of + (cert_file, key_file). + """ + pool_kwargs = {} + if verify: + cert_loc = None + # Allow self-specified cert location. + if verify is not True: + cert_loc = verify + if not cert_loc: + cert_loc = DEFAULT_CA_BUNDLE_PATH + if not cert_loc or not os.path.exists(cert_loc): + raise IOError( + "Could not find a suitable TLS CA certificate bundle, " + "invalid path: {0}".format(cert_loc) + ) + + pool_kwargs['cert_reqs'] = 'CERT_REQUIRED' + if not os.path.isdir(cert_loc): + pool_kwargs['ca_certs'] = cert_loc + pool_kwargs['ca_cert_dir'] = None + else: + pool_kwargs['ca_cert_dir'] = cert_loc + pool_kwargs['ca_certs'] = None + else: + pool_kwargs['cert_reqs'] = 'CERT_NONE' + pool_kwargs['ca_certs'] = None + pool_kwargs['ca_cert_dir'] = None + if cert: + if not isinstance(cert, basestring): + pool_kwargs['cert_file'] = cert[0] + pool_kwargs['key_file'] = cert[1] + else: + pool_kwargs['cert_file'] = cert + pool_kwargs['key_file'] = None + cert_file = pool_kwargs['cert_file'] + key_file = pool_kwargs['key_file'] + if cert_file and not os.path.exists(cert_file): + raise IOError( + "Could not find the TLS certificate file, " + "invalid path: {0}".format(cert_file) + ) + + if key_file and not os.path.exists(key_file): + raise IOError( + "Could not find the TLS key file, " + "invalid path: {0}".format(key_file) + ) + + return pool_kwargs + + +class BaseAdapter(object): + """The Base Transport Adapter""" + + def __init__(self): + super(BaseAdapter, self).__init__() + + def send( + self, + request, + stream=False, + timeout=None, + verify=True, + cert=None, + proxies=None, + ): + """Sends PreparedRequest object. Returns Response object. + + :param request: The :class:`PreparedRequest ` being sent. + :param stream: (optional) Whether to stream the request content. + :param timeout: (optional) How long to wait for the server to send + data before giving up, as a float, or a :ref:`(connect timeout, + read timeout) ` tuple. + :type timeout: float or tuple + :param verify: (optional) Either a boolean, in which case it controls whether we verify + the server's TLS certificate, or a string, in which case it must be a path + to a CA bundle to use + :param cert: (optional) Any user-provided SSL certificate to be trusted. + :param proxies: (optional) The proxies dictionary to apply to the request. + """ + raise NotImplementedError + + def close(self): + """Cleans up adapter specific items.""" + raise NotImplementedError + + +class HTTPAdapter(BaseAdapter): + """The built-in HTTP Adapter for urllib3. + + Provides a general-case interface for Requests sessions to contact HTTP and + HTTPS urls by implementing the Transport Adapter interface. This class will + usually be created by the :class:`Session ` class under the + covers. + + :param pool_connections: The number of urllib3 connection pools to cache. + :param pool_maxsize: The maximum number of connections to save in the pool. + :param max_retries: The maximum number of retries each connection + should attempt. Note, this applies only to failed DNS lookups, socket + connections and connection timeouts, never to requests where data has + made it to the server. By default, Requests does not retry failed + connections. If you need granular control over the conditions under + which we retry a request, import urllib3's ``Retry`` class and pass + that instead. + :param pool_block: Whether the connection pool should block for connections. + + Usage:: + + >>> import requests + >>> s = requests.Session() + >>> a = requests.adapters.HTTPAdapter(max_retries=3) + >>> s.mount('http://', a) + """ + __attrs__ = [ + 'max_retries', + 'config', + '_pool_connections', + '_pool_maxsize', + '_pool_block', + ] + + def __init__( + self, + pool_connections=DEFAULT_POOLSIZE, + pool_maxsize=DEFAULT_POOLSIZE, + max_retries=DEFAULT_RETRIES, + pool_block=DEFAULT_POOLBLOCK, + ): + if max_retries == DEFAULT_RETRIES: + self.max_retries = Retry(0, read=False) + else: + self.max_retries = Retry.from_int(max_retries) + self.config = {} + self.proxy_manager = {} + super(HTTPAdapter, self).__init__() + self._pool_connections = pool_connections + self._pool_maxsize = pool_maxsize + self._pool_block = pool_block + self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) + + def __getstate__(self): + return {attr: getattr(self, attr, None) for attr in self.__attrs__} + + def __setstate__(self, state): + # Can't handle by adding 'proxy_manager' to self.__attrs__ because + # self.poolmanager uses a lambda function, which isn't pickleable. + self.proxy_manager = {} + self.config = {} + for attr, value in state.items(): + setattr(self, attr, value) + self.init_poolmanager( + self._pool_connections, self._pool_maxsize, block=self._pool_block + ) + + def init_poolmanager( + self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs + ): + """Initializes a urllib3 PoolManager. + + This method should not be called from user code, and is only + exposed for use when subclassing the + :class:`HTTPAdapter `. + + :param connections: The number of urllib3 connection pools to cache. + :param maxsize: The maximum number of connections to save in the pool. + :param block: Block when no free connections are available. + :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager. + """ + # save these values for pickling + self._pool_connections = connections + self._pool_maxsize = maxsize + self._pool_block = block + self.poolmanager = PoolManager( + num_pools=connections, + maxsize=maxsize, + block=block, + strict=True, + **pool_kwargs, + ) + + def proxy_manager_for(self, proxy, **proxy_kwargs): + """Return urllib3 ProxyManager for the given proxy. + + This method should not be called from user code, and is only + exposed for use when subclassing the + :class:`HTTPAdapter `. + + :param proxy: The proxy to return a urllib3 ProxyManager for. + :param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager. + :returns: ProxyManager + :rtype: urllib3.ProxyManager + """ + if proxy in self.proxy_manager: + manager = self.proxy_manager[proxy] + elif proxy.lower().startswith('socks'): + username, password = get_auth_from_url(proxy) + manager = self.proxy_manager[proxy] = SOCKSProxyManager( + proxy, + username=username, + password=password, + num_pools=self._pool_connections, + maxsize=self._pool_maxsize, + block=self._pool_block, + **proxy_kwargs, + ) + else: + proxy_headers = self.proxy_headers(proxy) + manager = self.proxy_manager[proxy] = proxy_from_url( + proxy, + proxy_headers=proxy_headers, + num_pools=self._pool_connections, + maxsize=self._pool_maxsize, + block=self._pool_block, + **proxy_kwargs, + ) + return manager + + def build_response(self, req, resp): + """Builds a :class:`Response ` object from a urllib3 + response. This should not be called from user code, and is only exposed + for use when subclassing the + :class:`HTTPAdapter ` + + :param req: The :class:`PreparedRequest ` used to generate the response. + :param resp: The urllib3 response object. + :rtype: requests.Response + """ + response = Response() + # Fallback to None if there's no status_code, for whatever reason. + response.status_code = getattr(resp, 'status', None) + # Make headers case-insensitive. + response.headers = HTTPHeaderDict(getattr(resp, 'headers', {})) + # Set encoding. + response.encoding = get_encoding_from_headers(response.headers) + response.raw = resp + response.reason = response.raw.reason + if isinstance(req.url, bytes): + response.url = req.url.decode('utf-8') + else: + response.url = req.url + # Add new cookies from the server. + extract_cookies_to_jar(response.cookies, req, resp) + # Give the Response some context. + response.request = req + response.connection = self + return response + + def get_connection(self, url, proxies=None, verify=None, cert=None): + """Returns a urllib3 connection for the given URL. This should not be + called from user code, and is only exposed for use when subclassing the + :class:`HTTPAdapter `. + + :param url: The URL to connect to. + :param proxies: (optional) A Requests-style dictionary of proxies used on this request. + :rtype: urllib3.ConnectionPool + """ + pool_kwargs = _pool_kwargs(verify, cert) + proxy = select_proxy(url, proxies) + if proxy: + proxy = prepend_scheme_if_needed(proxy, 'http') + proxy_manager = self.proxy_manager_for(proxy) + conn = proxy_manager.connection_from_url( + url, pool_kwargs=pool_kwargs + ) + else: + # Only scheme should be lower case + parsed = urlparse(url) + url = parsed.geturl() + conn = self.poolmanager.connection_from_url( + url, pool_kwargs=pool_kwargs + ) + return conn + + def close(self): + """Disposes of any internal state. + + Currently, this closes the PoolManager and any active ProxyManager, + which closes any pooled connections. + """ + self.poolmanager.clear() + for proxy in self.proxy_manager.values(): + proxy.clear() + + def request_url(self, request, proxies): + """Obtain the url to use when making the final request. + + If the message is being sent through a HTTP proxy, the full URL has to + be used. Otherwise, we should only use the path portion of the URL. + + This should not be called from user code, and is only exposed for use + when subclassing the + :class:`HTTPAdapter `. + + :param request: The :class:`PreparedRequest ` being sent. + :param proxies: A dictionary of schemes or schemes and hosts to proxy URLs. + :rtype: str + """ + proxy = select_proxy(request.url, proxies) + scheme = urlparse(request.url).scheme + is_proxied_http_request = (proxy and scheme != 'https') + using_socks_proxy = False + if proxy: + proxy_scheme = urlparse(proxy).scheme.lower() + using_socks_proxy = proxy_scheme.startswith('socks') + url = request.path_url + if is_proxied_http_request and not using_socks_proxy: + url = urldefragauth(request.url) + return url + + def add_headers(self, request, **kwargs): + """Add any headers needed by the connection. As of v2.0 this does + nothing by default, but is left for overriding by users that subclass + the :class:`HTTPAdapter `. + + This should not be called from user code, and is only exposed for use + when subclassing the + :class:`HTTPAdapter `. + + :param request: The :class:`PreparedRequest ` to add headers to. + :param kwargs: The keyword arguments from the call to send(). + """ + pass + + def proxy_headers(self, proxy): + """Returns a dictionary of the headers to add to any request sent + through a proxy. This works with urllib3 magic to ensure that they are + correctly sent to the proxy, rather than in a tunnelled request if + CONNECT is being used. + + This should not be called from user code, and is only exposed for use + when subclassing the + :class:`HTTPAdapter `. + + :param proxies: The url of the proxy being used for this request. + :rtype: dict + """ + headers = {} + username, password = get_auth_from_url(proxy) + if username: + headers['Proxy-Authorization'] = _basic_auth_str( + username, password + ) + return headers + + def send( + self, + request, + stream=False, + timeout=None, + verify=True, + cert=None, + proxies=None, + ): + """Sends PreparedRequest object. Returns Response object. + + :param request: The :class:`PreparedRequest ` being sent. + :param stream: (optional) Whether to stream the request content. + :param timeout: (optional) How long to wait for the server to send + data before giving up, as a float, or a :ref:`(connect timeout, + read timeout) ` tuple. + :type timeout: float or tuple or urllib3 Timeout object + :param verify: (optional) Either a boolean, in which case it controls whether + we verify the server's TLS certificate, or a string, in which case it + must be a path to a CA bundle to use + :param cert: (optional) Any user-provided SSL certificate to be trusted. + :param proxies: (optional) The proxies dictionary to apply to the request. + :rtype: requests.Response + """ + conn = self.get_connection(request.url, proxies, verify, cert) + url = self.request_url(request, proxies) + self.add_headers(request) + chunked = not ( + request.body is None or 'Content-Length' in request.headers + ) + if isinstance(timeout, tuple): + try: + connect, read = timeout + timeout = TimeoutSauce(connect=connect, read=read) + except ValueError as e: + # this may raise a string formatting error. + err = ( + "Invalid timeout {0}. Pass a (connect, read) " + "timeout tuple, or a single float to set " + "both timeouts to the same value".format(timeout) + ) + raise ValueError(err) + + elif isinstance(timeout, TimeoutSauce): + pass + else: + timeout = TimeoutSauce(connect=timeout, read=timeout) + try: + if not chunked: + resp = core.blocking_request( + method=request.method, + url=url, + body=request.body, + headers=request.headers, + redirect=False, + assert_same_host=False, + preload_content=False, + decode_content=False, + retries=self.max_retries, + timeout=timeout, + enforce_content_length=True, + pool=conn + ) + # Send the request. + else: + if hasattr(conn, 'proxy_pool'): + conn = conn.proxy_pool + low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT) + try: + low_conn.putrequest( + request.method, url, skip_accept_encoding=True + ) + for header, value in request.headers.items(): + low_conn.putheader(header, value) + low_conn.endheaders() + for i in request.body: + chunk_size = len(i) + if chunk_size == 0: + continue + + low_conn.send(hex(chunk_size)[2:].encode('utf-8')) + low_conn.send(b'\r\n') + low_conn.send(i) + low_conn.send(b'\r\n') + low_conn.send(b'0\r\n\r\n') + # Receive the response from the server + try: + # For Python 2.7, use buffering of HTTP responses + r = low_conn.getresponse(buffering=True) + except TypeError: + # For Python 3.3+ versions, this is the default + r = low_conn.getresponse() + resp = HTTPResponse.from_httplib( + r, + pool=conn, + connection=low_conn, + preload_content=False, + decode_content=False, + enforce_content_length=True, + request_method=request.method, + ) + except: + # If we hit any problems here, clean up the connection. + # Then, reraise so that we can handle the actual exception. + low_conn.close() + raise + + except (ProtocolError, socket.error) as err: + raise ConnectionError(err, request=request) + + except MaxRetryError as e: + if isinstance(e.reason, ConnectTimeoutError): + # TODO: Remove this in 3.0.0: see #2811 + if not isinstance(e.reason, NewConnectionError): + raise ConnectTimeout(e, request=request) + + if isinstance(e.reason, ResponseError): + raise RetryError(e, request=request) + + if isinstance(e.reason, _ProxyError): + raise ProxyError(e, request=request) + + if isinstance(e.reason, _SSLError): + # This branch is for urllib3 v1.22 and later. + raise SSLError(e, request=request) + + raise ConnectionError(e, request=request) + + except ClosedPoolError as e: + raise ConnectionError(e, request=request) + + except _ProxyError as e: + raise ProxyError(e) + + except (_SSLError, _HTTPError) as e: + if isinstance(e, _SSLError): + # This branch is for urllib3 versions earlier than v1.22 + raise SSLError(e, request=request) + + elif isinstance(e, ReadTimeoutError): + raise ReadTimeout(e, request=request) + + else: + raise + + return self.build_response(request, resp) + + +class AsyncHTTPAdapter(HTTPAdapter): + """docstring for AsyncHTTPAdapter""" + def __init__(self, backend=None, *args, **kwargs): + self.backend = backend or TrioBackend() + super(AsyncHTTPAdapter, self).__init__(*args, **kwargs) + + async def build_response(self, req, resp): + """Builds a :class:`Response ` object from a urllib3 + response. This should not be called from user code, and is only exposed + for use when subclassing the + :class:`HTTPAdapter ` + + :param req: The :class:`PreparedRequest ` used to generate the response. + :param resp: The urllib3 response object. + :rtype: requests.Response + """ + response = AsyncResponse() + # Fallback to None if there's no status_code, for whatever reason. + response.status_code = getattr(resp, 'status', None) + # Make headers case-insensitive. + response.headers = HTTPHeaderDict(getattr(resp, 'headers', {})) + # Set encoding. + response.encoding = get_encoding_from_headers(response.headers) + response.raw = resp + response.reason = response.raw.reason + if isinstance(req.url, bytes): + response.url = req.url.decode('utf-8') + else: + response.url = req.url + # Add new cookies from the server. + extract_cookies_to_jar(response.cookies, req, resp) + # Give the Response some context. + response.request = req + response.connection = self + return response + + def init_poolmanager( + self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs + ): + """Initializes a urllib3 PoolManager. + + This method should not be called from user code, and is only + exposed for use when subclassing the + :class:`HTTPAdapter `. + + :param connections: The number of urllib3 connection pools to cache. + :param maxsize: The maximum number of connections to save in the pool. + :param block: Block when no free connections are available. + :param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager. + """ + # save these values for pickling + self._pool_connections = connections + self._pool_maxsize = maxsize + self._pool_block = block + self.poolmanager = AsyncPoolManager( + num_pools=connections, + maxsize=maxsize, + block=block, + strict=True, + backend=self.backend, + **pool_kwargs, + ) + + def get_connection(self, url, proxies=None, verify=None, cert=None): + """Returns a urllib3 connection for the given URL. This should not be + called from user code, and is only exposed for use when subclassing the + :class:`HTTPAdapter `. + + :param url: The URL to connect to. + :param proxies: (optional) A Requests-style dictionary of proxies used on this request. + :rtype: urllib3.ConnectionPool + """ + pool_kwargs = _pool_kwargs(verify, cert) + proxy = select_proxy(url, proxies) + if proxy: + proxy = prepend_scheme_if_needed(proxy, 'http') + proxy_manager = self.proxy_manager_for(proxy) + conn = proxy_manager.connection_from_url( + url, pool_kwargs=pool_kwargs + ) + else: + # Only scheme should be lower case + parsed = urlparse(url) + url = parsed.geturl() + conn = self.poolmanager.connection_from_url( + url, pool_kwargs=pool_kwargs + ) + return conn + + def close(self): + """Disposes of any internal state. + + Currently, this closes the PoolManager and any active ProxyManager, + which closes any pooled connections. + """ + self.poolmanager.clear() + for proxy in self.proxy_manager.values(): + proxy.clear() + pass + + async def send( + self, + request, + stream=False, + timeout=None, + verify=True, + cert=None, + proxies=None, + ): + """Sends PreparedRequest object. Returns Response object. + + :param request: The :class:`PreparedRequest ` being sent. + :param stream: (optional) Whether to stream the request content. + :param timeout: (optional) How long to wait for the server to send + data before giving up, as a float, or a :ref:`(connect timeout, + read timeout) ` tuple. + :type timeout: float or tuple or urllib3 Timeout object + :param verify: (optional) Either a boolean, in which case it controls whether + we verify the server's TLS certificate, or a string, in which case it + must be a path to a CA bundle to use + :param cert: (optional) Any user-provided SSL certificate to be trusted. + :param proxies: (optional) The proxies dictionary to apply to the request. + :rtype: requests.Response + """ + conn = self.get_connection(request.url, proxies, verify, cert) + + url = self.request_url(request, proxies) + self.add_headers(request) + chunked = not ( + request.body is None or 'Content-Length' in request.headers + ) + if isinstance(timeout, tuple): + try: + connect, read = timeout + timeout = TimeoutSauce(connect=connect, read=read) + except ValueError as e: + # this may raise a string formatting error. + err = ( + "Invalid timeout {0}. Pass a (connect, read) " + "timeout tuple, or a single float to set " + "both timeouts to the same value".format(timeout) + ) + raise ValueError(err) + + elif isinstance(timeout, TimeoutSauce): + pass + else: + timeout = TimeoutSauce(connect=timeout, read=timeout) + try: + if not chunked: + resp = await core.request( + method=request.method, + url=url, + body=request.body, + headers=request.headers, + redirect=False, + assert_same_host=False, + preload_content=False, + decode_content=False, + retries=self.max_retries, + timeout=timeout, + enforce_content_length=True, + pool=conn + ) + + # Send the request. + else: + if hasattr(conn, 'proxy_pool'): + conn = conn.proxy_pool + low_conn = conn._get_conn(timeout=DEFAULT_POOL_TIMEOUT) + try: + low_conn.putrequest( + request.method, url, skip_accept_encoding=True + ) + for header, value in request.headers.items(): + low_conn.putheader(header, value) + low_conn.endheaders() + for i in request.body: + chunk_size = len(i) + if chunk_size == 0: + continue + + low_conn.send(hex(chunk_size)[2:].encode('utf-8')) + low_conn.send(b'\r\n') + low_conn.send(i) + low_conn.send(b'\r\n') + low_conn.send(b'0\r\n\r\n') + # Receive the response from the server + try: + # For Python 2.7, use buffering of HTTP responses + r = alow_conn.getresponse(buffering=True) + except TypeError: + # For Python 3.3+ versions, this is the default + r = low_conn.getresponse() + resp = HTTPResponse.from_httplib( + r, + pool=conn, + connection=low_conn, + preload_content=False, + decode_content=False, + enforce_content_length=True, + request_method=request.method, + ) + except: + # If we hit any problems here, clean up the connection. + # Then, reraise so that we can handle the actual exception. + low_conn.close() + raise + + except (ProtocolError, socket.error) as err: + raise ConnectionError(err, request=request) + + except MaxRetryError as e: + if isinstance(e.reason, ConnectTimeoutError): + # TODO: Remove this in 3.0.0: see #2811 + if not isinstance(e.reason, NewConnectionError): + raise ConnectTimeout(e, request=request) + + if isinstance(e.reason, ResponseError): + raise RetryError(e, request=request) + + if isinstance(e.reason, _ProxyError): + raise ProxyError(e, request=request) + + if isinstance(e.reason, _SSLError): + # This branch is for urllib3 v1.22 and later. + raise SSLError(e, request=request) + + raise ConnectionError(e, request=request) + + except ClosedPoolError as e: + raise ConnectionError(e, request=request) + + except _ProxyError as e: + raise ProxyError(e) + + except (_SSLError, _HTTPError) as e: + if isinstance(e, _SSLError): + # This branch is for urllib3 versions earlier than v1.22 + raise SSLError(e, request=request) + + elif isinstance(e, ReadTimeoutError): + raise ReadTimeout(e, request=request) + + else: + raise + + return await self.build_response(request, resp) diff --git a/requests/api.py b/requests3/api.py similarity index 74% rename from requests/api.py rename to requests3/api.py index ef71d075..b6265ba5 100644 --- a/requests/api.py +++ b/requests3/api.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.api ~~~~~~~~~~~~ @@ -11,18 +10,20 @@ This module implements the Requests API. """ from . import sessions +from . import types -def request(method, url, **kwargs): +def request( + method: types.Method, url: types.URL, *, session: types.Session = None, **kwargs +) -> types.Response: """Constructs and sends a :class:`Request `. :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary, list of tuples or bytes to send - in the query string for the :class:`Request`. - :param data: (optional) Dictionary, list of tuples, bytes, or file-like - object to send in the body of the :class:`Request`. - :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. + :param session: :class:`Session` object to use for this request. If none is given, one will be provided. + :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. + :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param json: (optional) json data to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload. @@ -52,15 +53,15 @@ def request(method, url, **kwargs): >>> req = requests.request('GET', 'https://httpbin.org/get') """ - # By using the 'with' statement we are sure the session is closed, thus we # avoid leaving sockets open which can trigger a ResourceWarning in some # cases, and look like a memory leak in others. - with sessions.Session() as session: + session = sessions.Session() if session is None else session + with session: return session.request(method=method, url=url, **kwargs) -def get(url, params=None, **kwargs): +def get(url: types.URL, *, params: types.Params = None, **kwargs) -> types.Response: r"""Sends a GET request. :param url: URL for the new :class:`Request` object. @@ -70,12 +71,11 @@ def get(url, params=None, **kwargs): :return: :class:`Response ` object :rtype: requests.Response """ - - kwargs.setdefault('allow_redirects', True) - return request('get', url, params=params, **kwargs) + kwargs.setdefault("allow_redirects", True) + return request("get", url, params=params, **kwargs) -def options(url, **kwargs): +def options(url: types.URL, **kwargs) -> types.Response: r"""Sends an OPTIONS request. :param url: URL for the new :class:`Request` object. @@ -83,12 +83,11 @@ def options(url, **kwargs): :return: :class:`Response ` object :rtype: requests.Response """ - - kwargs.setdefault('allow_redirects', True) - return request('options', url, **kwargs) + kwargs.setdefault("allow_redirects", True) + return request("options", url, **kwargs) -def head(url, **kwargs): +def head(url: types.URL, **kwargs) -> types.Response: r"""Sends a HEAD request. :param url: URL for the new :class:`Request` object. @@ -96,12 +95,13 @@ def head(url, **kwargs): :return: :class:`Response ` object :rtype: requests.Response """ - - kwargs.setdefault('allow_redirects', False) - return request('head', url, **kwargs) + kwargs.setdefault("allow_redirects", False) + return request("head", url, **kwargs) -def post(url, data=None, json=None, **kwargs): +def post( + url: types.URL, *, data: types.Data = None, json: types.JSON = None, **kwargs +) -> types.Response: r"""Sends a POST request. :param url: URL for the new :class:`Request` object. @@ -112,11 +112,10 @@ def post(url, data=None, json=None, **kwargs): :return: :class:`Response ` object :rtype: requests.Response """ - - return request('post', url, data=data, json=json, **kwargs) + return request("post", url, data=data, json=json, **kwargs) -def put(url, data=None, **kwargs): +def put(url: types.URL, *, data: types.Data = None, **kwargs) -> types.Response: r"""Sends a PUT request. :param url: URL for the new :class:`Request` object. @@ -127,11 +126,10 @@ def put(url, data=None, **kwargs): :return: :class:`Response ` object :rtype: requests.Response """ - - return request('put', url, data=data, **kwargs) + return request("put", url, data=data, **kwargs) -def patch(url, data=None, **kwargs): +def patch(url: types.URL, *, data: types.Data = None, **kwargs) -> types.Response: r"""Sends a PATCH request. :param url: URL for the new :class:`Request` object. @@ -142,11 +140,10 @@ def patch(url, data=None, **kwargs): :return: :class:`Response ` object :rtype: requests.Response """ - - return request('patch', url, data=data, **kwargs) + return request("patch", url, data=data, **kwargs) -def delete(url, **kwargs): +def delete(url: types.URL, **kwargs) -> types.Response: r"""Sends a DELETE request. :param url: URL for the new :class:`Request` object. @@ -154,5 +151,4 @@ def delete(url, **kwargs): :return: :class:`Response ` object :rtype: requests.Response """ - - return request('delete', url, **kwargs) + return request("delete", url, **kwargs) diff --git a/requests/auth.py b/requests3/auth.py similarity index 65% rename from requests/auth.py rename to requests3/auth.py index bdde51c7..afe0bd4f 100644 --- a/requests/auth.py +++ b/requests3/auth.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.auth ~~~~~~~~~~~~~ @@ -12,29 +11,20 @@ import re import time import hashlib import threading -import warnings from base64 import b64encode -from .compat import urlparse, str, basestring +from .basics import urlparse, str, basestring from .cookies import extract_cookies_to_jar from ._internal_utils import to_native_string from .utils import parse_dict_header -CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded' -CONTENT_TYPE_MULTI_PART = 'multipart/form-data' +CONTENT_TYPE_FORM_URLENCODED = "application/x-www-form-urlencoded" +CONTENT_TYPE_MULTI_PART = "multipart/form-data" def _basic_auth_str(username, password): """Returns a Basic Auth string.""" - - # "I want us to put a big-ol' comment on top of it that - # says that this behaviour is dumb but we need to preserve - # it because people are relying on it." - # - Lukasa - # - # These are here solely to maintain backwards compatibility - # for things like ints. This will be removed in 3.0.0. if not isinstance(username, basestring): warnings.warn( "Non-string usernames will no longer be supported in Requests " @@ -43,7 +33,6 @@ def _basic_auth_str(username, password): "problems.".format(username), category=DeprecationWarning, ) - username = str(username) if not isinstance(password, basestring): warnings.warn( @@ -53,19 +42,14 @@ def _basic_auth_str(username, password): "problems.".format(password), category=DeprecationWarning, ) - password = str(password) - # -- End Removal -- if isinstance(username, str): - username = username.encode('latin1') - + username = username.encode("latin1") if isinstance(password, str): - password = password.encode('latin1') - - authstr = 'Basic ' + to_native_string( - b64encode(b':'.join((username, password))).strip() + password = password.encode("latin1") + authstr = "Basic " + to_native_string( + b64encode(b":".join((username, password))).strip() ) - return authstr @@ -73,7 +57,7 @@ class AuthBase(object): """Base class that all auth implementations derive from""" def __call__(self, r): - raise NotImplementedError('Auth hooks must be callable.') + raise NotImplementedError("Auth hooks must be callable.") class HTTPBasicAuth(AuthBase): @@ -84,24 +68,18 @@ class HTTPBasicAuth(AuthBase): self.password = password def __eq__(self, other): - return all([ - self.username == getattr(other, 'username', None), - self.password == getattr(other, 'password', None) - ]) + return all( + [ + self.username == getattr(other, "username", None), + self.password == getattr(other, "password", None), + ] + ) def __ne__(self, other): return not self == other def __call__(self, r): - r.headers['Authorization'] = _basic_auth_str(self.username, self.password) - return r - - -class HTTPProxyAuth(HTTPBasicAuth): - """Attaches HTTP Proxy Authentication to a given Request object.""" - - def __call__(self, r): - r.headers['Proxy-Authorization'] = _basic_auth_str(self.username, self.password) + r.headers["Authorization"] = _basic_auth_str(self.username, self.password) return r @@ -116,9 +94,9 @@ class HTTPDigestAuth(AuthBase): def init_per_thread_state(self): # Ensure state is initialized just once per-thread - if not hasattr(self._thread_local, 'init'): + if not hasattr(self._thread_local, "init"): self._thread_local.init = True - self._thread_local.last_nonce = '' + self._thread_local.last_nonce = "" self._thread_local.nonce_count = 0 self._thread_local.chal = {} self._thread_local.pos = None @@ -128,93 +106,96 @@ class HTTPDigestAuth(AuthBase): """ :rtype: str """ - - realm = self._thread_local.chal['realm'] - nonce = self._thread_local.chal['nonce'] - qop = self._thread_local.chal.get('qop') - algorithm = self._thread_local.chal.get('algorithm') - opaque = self._thread_local.chal.get('opaque') + realm = self._thread_local.chal["realm"] + nonce = self._thread_local.chal["nonce"] + qop = self._thread_local.chal.get("qop") + algorithm = self._thread_local.chal.get("algorithm") + opaque = self._thread_local.chal.get("opaque") hash_utf8 = None - if algorithm is None: - _algorithm = 'MD5' + _algorithm = "MD5" else: _algorithm = algorithm.upper() # lambdas assume digest modules are imported at the top level - if _algorithm == 'MD5' or _algorithm == 'MD5-SESS': + if _algorithm == "MD5" or _algorithm == "MD5-SESS": + def md5_utf8(x): if isinstance(x, str): - x = x.encode('utf-8') + x = x.encode("utf-8") return hashlib.md5(x).hexdigest() + hash_utf8 = md5_utf8 - elif _algorithm == 'SHA': + elif _algorithm == "SHA": + def sha_utf8(x): if isinstance(x, str): - x = x.encode('utf-8') + x = x.encode("utf-8") return hashlib.sha1(x).hexdigest() + hash_utf8 = sha_utf8 - elif _algorithm == 'SHA-256': + elif _algorithm == "SHA-256": + def sha256_utf8(x): if isinstance(x, str): - x = x.encode('utf-8') + x = x.encode("utf-8") return hashlib.sha256(x).hexdigest() + hash_utf8 = sha256_utf8 - elif _algorithm == 'SHA-512': + elif _algorithm == "SHA-512": + def sha512_utf8(x): if isinstance(x, str): - x = x.encode('utf-8') + x = x.encode("utf-8") return hashlib.sha512(x).hexdigest() + hash_utf8 = sha512_utf8 + hash_utf8 = sha_utf8 KD = lambda s, d: hash_utf8("%s:%s" % (s, d)) - if hash_utf8 is None: return None # XXX not implemented yet entdig = None p_parsed = urlparse(url) - #: path is request-uri defined in RFC 2616 which should not be empty + # : path is request-uri defined in RFC 2616 which should not be empty path = p_parsed.path or "/" if p_parsed.query: - path += '?' + p_parsed.query - - A1 = '%s:%s:%s' % (self.username, realm, self.password) - A2 = '%s:%s' % (method, path) - + path += "?" + p_parsed.query + A1 = "%s:%s:%s" % (self.username, realm, self.password) + A2 = "%s:%s" % (method, path) HA1 = hash_utf8(A1) HA2 = hash_utf8(A2) - if nonce == self._thread_local.last_nonce: self._thread_local.nonce_count += 1 else: self._thread_local.nonce_count = 1 - ncvalue = '%08x' % self._thread_local.nonce_count - s = str(self._thread_local.nonce_count).encode('utf-8') - s += nonce.encode('utf-8') - s += time.ctime().encode('utf-8') + ncvalue = "%08x" % self._thread_local.nonce_count + s = str(self._thread_local.nonce_count).encode("utf-8") + s += nonce.encode("utf-8") + s += time.ctime().encode("utf-8") s += os.urandom(8) - - cnonce = (hashlib.sha1(s).hexdigest()[:16]) - if _algorithm == 'MD5-SESS': - HA1 = hash_utf8('%s:%s:%s' % (HA1, nonce, cnonce)) - + cnonce = hashlib.sha1(s).hexdigest()[:16] + if _algorithm == "MD5-SESS": + HA1 = hash_utf8("%s:%s:%s" % (HA1, nonce, cnonce)) if not qop: respdig = KD(HA1, "%s:%s" % (nonce, HA2)) - elif qop == 'auth' or 'auth' in qop.split(','): - noncebit = "%s:%s:%s:%s:%s" % ( - nonce, ncvalue, cnonce, 'auth', HA2 - ) + elif qop == "auth" or "auth" in qop.split(","): + noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, "auth", HA2) respdig = KD(HA1, noncebit) else: # XXX handle auth-int. return None self._thread_local.last_nonce = nonce - # XXX should the partial digests be encoded too? - base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' \ - 'response="%s"' % (self.username, realm, nonce, path, respdig) + base = 'username="%s", realm="%s", nonce="%s", uri="%s", ' 'response="%s"' % ( + self.username, + realm, + nonce, + path, + respdig, + ) if opaque: base += ', opaque="%s"' % opaque if algorithm: @@ -223,8 +204,7 @@ class HTTPDigestAuth(AuthBase): base += ', digest="%s"' % entdig if qop: base += ', qop="auth", nc=%s, cnonce="%s"' % (ncvalue, cnonce) - - return 'Digest %s' % (base) + return "Digest %s" % (base) def handle_redirect(self, r, **kwargs): """Reset num_401_calls counter on redirects.""" @@ -237,7 +217,6 @@ class HTTPDigestAuth(AuthBase): :rtype: requests.Response """ - # If response is not 4xx, do not auth # See https://github.com/requests/requests/issues/3772 if not 400 <= r.status_code < 500: @@ -248,14 +227,11 @@ class HTTPDigestAuth(AuthBase): # Rewind the file position indicator of the body to where # it was to resend the request. r.request.body.seek(self._thread_local.pos) - s_auth = r.headers.get('www-authenticate', '') - - if 'digest' in s_auth.lower() and self._thread_local.num_401_calls < 2: - + s_auth = r.headers.get("www-authenticate", "") + if "digest" in s_auth.lower() and self._thread_local.num_401_calls < 2: self._thread_local.num_401_calls += 1 - pat = re.compile(r'digest ', flags=re.IGNORECASE) - self._thread_local.chal = parse_dict_header(pat.sub('', s_auth, count=1)) - + pat = re.compile(r"digest ", flags=re.IGNORECASE) + self._thread_local.chal = parse_dict_header(pat.sub("", s_auth, count=1)) # Consume content and release the original connection # to allow our new request to reuse the same one. r.content @@ -263,13 +239,12 @@ class HTTPDigestAuth(AuthBase): prep = r.request.copy() extract_cookies_to_jar(prep._cookies, r.request, r.raw) prep.prepare_cookies(prep._cookies) - - prep.headers['Authorization'] = self.build_digest_header( - prep.method, prep.url) + prep.headers["Authorization"] = self.build_digest_header( + prep.method, prep.url + ) _r = r.connection.send(prep, **kwargs) _r.history.append(r) _r.request = prep - return _r self._thread_local.num_401_calls = 1 @@ -280,7 +255,7 @@ class HTTPDigestAuth(AuthBase): self.init_per_thread_state() # If we have a saved nonce, skip the 401 if self._thread_local.last_nonce: - r.headers['Authorization'] = self.build_digest_header(r.method, r.url) + r.headers["Authorization"] = self.build_digest_header(r.method, r.url) try: self._thread_local.pos = r.body.tell() except AttributeError: @@ -289,17 +264,18 @@ class HTTPDigestAuth(AuthBase): # file position of the previous body. Ensure it's set to # None. self._thread_local.pos = None - r.register_hook('response', self.handle_401) - r.register_hook('response', self.handle_redirect) + r.register_hook("response", self.handle_401) + r.register_hook("response", self.handle_redirect) self._thread_local.num_401_calls = 1 - return r def __eq__(self, other): - return all([ - self.username == getattr(other, 'username', None), - self.password == getattr(other, 'password', None) - ]) + return all( + [ + self.username == getattr(other, "username", None), + self.password == getattr(other, "password", None), + ] + ) def __ne__(self, other): return not self == other diff --git a/requests3/basics.py b/requests3/basics.py new file mode 100644 index 00000000..30b3f46f --- /dev/null +++ b/requests3/basics.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +""" +requests.basics +~~~~~~~~~~~~~~~ + +This modules covers the basics. +""" + +import chardet + +import sys + +# --------- +# Specifics +# --------- +from urllib.parse import ( + urlparse, + urlunparse, + urljoin, + urlsplit, + urlencode, + quote, + unquote, + quote_plus, + unquote_plus, + urldefrag, +) +from urllib.request import ( + parse_http_list, + getproxies, + proxy_bypass, + proxy_bypass_environment, + getproxies_environment, +) +from http import cookiejar as cookielib +from http.cookies import Morsel +from io import StringIO +from collections import OrderedDict + +builtin_str = str # type: ignore +str = str # type: ignore +bytes = bytes # type: ignore +basestring = (str, bytes) +numeric_types = (int, float) +integer_types = (int,) diff --git a/requests/certs.py b/requests3/certs.py similarity index 99% rename from requests/certs.py rename to requests3/certs.py index d1a378d7..c811c194 100644 --- a/requests/certs.py +++ b/requests3/certs.py @@ -1,6 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - """ requests.certs ~~~~~~~~~~~~~~ diff --git a/requests/cookies.py b/requests3/cookies.py similarity index 92% rename from requests/cookies.py rename to requests3/cookies.py index 56fccd9c..163646d6 100644 --- a/requests/cookies.py +++ b/requests3/cookies.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.cookies ~~~~~~~~~~~~~~~~ @@ -14,12 +13,12 @@ import time import calendar from ._internal_utils import to_native_string -from .compat import cookielib, urlparse, urlunparse, Morsel, MutableMapping +from .basics import cookielib, urlparse, urlunparse, Morsel try: import threading except ImportError: - import dummy_threading as threading + import dummy_threading as threading # type: ignore class MockRequest(object): @@ -53,14 +52,21 @@ class MockRequest(object): # header if not self._r.headers.get('Host'): return self._r.url + # If they did set it, retrieve it and reconstruct the expected domain host = to_native_string(self._r.headers['Host'], encoding='utf-8') parsed = urlparse(self._r.url) # Reconstruct the URL as we expect it - return urlunparse([ - parsed.scheme, host, parsed.path, parsed.params, parsed.query, - parsed.fragment - ]) + return urlunparse( + [ + parsed.scheme, + host, + parsed.path, + parsed.params, + parsed.query, + parsed.fragment, + ] + ) def is_unverifiable(self): return True @@ -73,7 +79,9 @@ class MockRequest(object): def add_header(self, key, val): """cookielib has no legitimate use for this method; add it back if you find one.""" - raise NotImplementedError("Cookie headers should be added with add_unredirected_header()") + raise NotImplementedError( + "Cookie headers should be added with add_unredirected_header()" + ) def add_unredirected_header(self, name, value): self._new_headers[name] = value @@ -122,13 +130,15 @@ def extract_cookies_to_jar(jar, request, response): :param request: our own requests.Request object :param response: urllib3.HTTPResponse object """ - if not (hasattr(response, '_original_response') and - response._original_response): + if not ( + hasattr(response, '_original_response') and response._original_response + ): return + # the _original_response field is the wrapped httplib.HTTPResponse object, req = MockRequest(request) # pull out the HTTPMessage with the headers and put it in the mock: - res = MockResponse(response._original_response.msg) + res = MockResponse(response._original_response.headers) jar.extract_cookies(res, req) @@ -152,12 +162,14 @@ def remove_cookie_by_name(cookiejar, name, domain=None, path=None): for cookie in cookiejar: if cookie.name != name: continue + if domain is not None and domain != cookie.domain: continue + if path is not None and path != cookie.path: continue - clearables.append((cookie.domain, cookie.path, cookie.name)) + clearables.append((cookie.domain, cookie.path, cookie.name)) for domain, path, name in clearables: cookiejar.clear(domain, path, name) @@ -195,6 +207,7 @@ class RequestsCookieJar(cookielib.CookieJar, MutableMapping): """ try: return self._find_no_duplicates(name, domain, path) + except KeyError: return default @@ -205,7 +218,12 @@ class RequestsCookieJar(cookielib.CookieJar, MutableMapping): """ # support client code that unsets cookies by assignment of a None value: if value is None: - remove_cookie_by_name(self, name, domain=kwargs.get('domain'), path=kwargs.get('path')) + remove_cookie_by_name( + self, + name, + domain=kwargs.get('domain'), + path=kwargs.get('path'), + ) return if isinstance(value, Morsel): @@ -293,6 +311,7 @@ class RequestsCookieJar(cookielib.CookieJar, MutableMapping): for cookie in iter(self): if cookie.domain is not None and cookie.domain in domains: return True + domains.append(cookie.domain) return False # there is only one domain in jar @@ -315,6 +334,7 @@ class RequestsCookieJar(cookielib.CookieJar, MutableMapping): def __contains__(self, name): try: return super(RequestsCookieJar, self).__contains__(name) + except CookieConflictError: return True @@ -341,9 +361,15 @@ class RequestsCookieJar(cookielib.CookieJar, MutableMapping): remove_cookie_by_name(self, name) def set_cookie(self, cookie, *args, **kwargs): - if hasattr(cookie.value, 'startswith') and cookie.value.startswith('"') and cookie.value.endswith('"'): + if hasattr(cookie.value, 'startswith') and cookie.value.startswith( + '"' + ) and cookie.value.endswith( + '"' + ): cookie.value = cookie.value.replace('\\"', '') - return super(RequestsCookieJar, self).set_cookie(cookie, *args, **kwargs) + return super(RequestsCookieJar, self).set_cookie( + cookie, *args, **kwargs + ) def update(self, other): """Updates this jar with cookies from another CookieJar or dict-like""" @@ -391,11 +417,15 @@ class RequestsCookieJar(cookielib.CookieJar, MutableMapping): if domain is None or cookie.domain == domain: if path is None or cookie.path == path: if toReturn is not None: # if there are multiple cookies that meet passed in criteria - raise CookieConflictError('There are multiple cookies with name, %r' % (name)) - toReturn = cookie.value # we will eventually return this as long as no cookie conflict + raise CookieConflictError( + 'There are multiple cookies with name, %r' % + (name) + ) + toReturn = cookie.value # we will eventually return this as long as no cookie conflict if toReturn: return toReturn + raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path)) def __getstate__(self): @@ -413,8 +443,7 @@ class RequestsCookieJar(cookielib.CookieJar, MutableMapping): def copy(self): """Return a copy of this RequestsCookieJar.""" - new_cj = RequestsCookieJar() - new_cj.set_policy(self.get_policy()) + new_cj = RequestsCookieJar(self._policy) new_cj.update(self) return new_cj @@ -430,6 +459,7 @@ def _copy_cookie_jar(jar): if hasattr(jar, 'copy'): # We're dealing with an instance of RequestsCookieJar return jar.copy() + # We're dealing with a generic CookieJar instance new_jar = copy.copy(jar) new_jar.clear() @@ -459,7 +489,6 @@ def create_cookie(name, value, **kwargs): 'rest': {'HttpOnly': None}, 'rfc2109': False, } - badargs = set(kwargs) - set(result) if badargs: err = 'create_cookie() got unexpected keyword arguments: %s' @@ -470,19 +499,18 @@ def create_cookie(name, value, **kwargs): result['domain_specified'] = bool(result['domain']) result['domain_initial_dot'] = result['domain'].startswith('.') result['path_specified'] = bool(result['path']) - return cookielib.Cookie(**result) def morsel_to_cookie(morsel): """Convert a Morsel object into a Cookie containing the one k/v pair.""" - expires = None if morsel['max-age']: try: expires = int(time.time() + int(morsel['max-age'])) except ValueError: raise TypeError('max-age: %s must be integer' % morsel['max-age']) + elif morsel['expires']: time_template = '%a, %d-%b-%Y %H:%M:%S GMT' expires = calendar.timegm( @@ -516,13 +544,11 @@ def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True): """ if cookiejar is None: cookiejar = RequestsCookieJar() - if cookie_dict is not None: names_from_jar = [cookie.name for cookie in cookiejar] for name in cookie_dict: if overwrite or (name not in names_from_jar): cookiejar.set_cookie(create_cookie(name, cookie_dict[name])) - return cookiejar @@ -538,12 +564,12 @@ def merge_cookies(cookiejar, cookies): if isinstance(cookies, dict): cookiejar = cookiejar_from_dict( - cookies, cookiejar=cookiejar, overwrite=False) + cookies, cookiejar=cookiejar, overwrite=False + ) elif isinstance(cookies, cookielib.CookieJar): try: cookiejar.update(cookies) except AttributeError: for cookie_in_jar in cookies: cookiejar.set_cookie(cookie_in_jar) - return cookiejar diff --git a/requests3/core/__init__.py b/requests3/core/__init__.py new file mode 100644 index 00000000..77375bc2 --- /dev/null +++ b/requests3/core/__init__.py @@ -0,0 +1,3 @@ +from .api import AsyncPoolManager +from .api import request, blocking_request +from .import _http diff --git a/requests3/core/_http/__init__.py b/requests3/core/_http/__init__.py new file mode 100644 index 00000000..362725be --- /dev/null +++ b/requests3/core/_http/__init__.py @@ -0,0 +1,111 @@ +""" +urllib3 - Thread-safe connection pooling and re-using. +""" +from __future__ import absolute_import +import warnings + +from .connectionpool import ( + HTTPConnectionPool, + HTTPSConnectionPool, + connection_from_url +) + +from . import exceptions +from .filepost import encode_multipart_formdata +from .poolmanager import PoolManager, ProxyManager, proxy_from_url +from .response import HTTPResponse +from .util.request import make_headers +from .util.url import get_host +from .util.timeout import Timeout +from .util.retry import Retry + + +# Set default logging handler to avoid "No handler found" warnings. +import logging +try: # Python 2.7+ + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + +__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' +__license__ = 'MIT' +__version__ = '2.0.dev0+bleach.spike.proof.of.concept.dont.use' + +__all__ = [ + 'HTTPConnectionPool', + 'HTTPSConnectionPool', + 'PoolManager', + 'ProxyManager', + 'HTTPResponse', + 'Retry', + 'Timeout', + 'add_stderr_logger', + 'connection_from_url', + 'disable_warnings', + 'encode_multipart_formdata', + 'get_host', + 'make_headers', + 'proxy_from_url', +] + +# For now we only support async on 3.6, because we use async generators +import sys +if sys.version_info >= (3, 6): + from ._async.connectionpool import ( + HTTPConnectionPool as AsyncHTTPConnectionPool, + HTTPSConnectionPool as AsyncHTTPSConnectionPool) + from ._async.poolmanager import ( + PoolManager as AsyncPoolManager, + ProxyManager as AsyncProxyManager) + from ._async.response import HTTPResponse as AsyncHTTPResponse + __all__.extend( + ('AsyncHTTPConnectionPool', 'AsyncHTTPSConnectionPool', + 'AsyncPoolManager', 'AsyncProxyManager', 'AsyncHTTPResponse')) + + +logging.getLogger(__name__).addHandler(NullHandler()) + + +def add_stderr_logger(level=logging.DEBUG): + """ + Helper for quickly adding a StreamHandler to the logger. Useful for + debugging. + + Returns the handler after adding it. + """ + # This method needs to be in this __init__.py to get the __name__ correct + # even if urllib3 is vendored within another package. + logger = logging.getLogger(__name__) + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) + logger.addHandler(handler) + logger.setLevel(level) + logger.debug('Added a stderr logging handler to logger: %s', __name__) + return handler + + +# ... Clean up. +del NullHandler + + +# All warning filters *must* be appended unless you're really certain that they +# shouldn't be: otherwise, it's very hard for users to use most Python +# mechanisms to silence them. +# SecurityWarning's always go off by default. +warnings.simplefilter('always', exceptions.SecurityWarning, append=True) +# SubjectAltNameWarning's should go off once per host +warnings.simplefilter('default', exceptions.SubjectAltNameWarning, append=True) +# InsecurePlatformWarning's don't vary between requests, so we keep it default. +warnings.simplefilter('default', exceptions.InsecurePlatformWarning, + append=True) +# SNIMissingWarnings should go off only once. +warnings.simplefilter('default', exceptions.SNIMissingWarning, append=True) + + +def disable_warnings(category=exceptions.HTTPWarning): + """ + Helper for quickly disabling all urllib3 warnings. + """ + warnings.simplefilter('ignore', category) diff --git a/requests3/core/_http/_async/__init__.py b/requests3/core/_http/_async/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/requests3/core/_http/_async/connection.py b/requests3/core/_http/_async/connection.py new file mode 100644 index 00000000..934bb516 --- /dev/null +++ b/requests3/core/_http/_async/connection.py @@ -0,0 +1,526 @@ +# -*- coding: utf-8 -*- +""" +This module implements the connection management logic. + +Unlike in http.client, the connection here is an object that is responsible +for a very small number of tasks: + + 1. Serializing/deserializing data to/from the network. + 2. Being able to do basic parsing of HTTP and maintaining the framing. + 3. Understanding connection state. + +This object knows very little about the semantics of HTTP in terms of how to +construct HTTP requests and responses. It mostly manages the socket itself. +""" +from __future__ import absolute_import + +import collections +import datetime +import socket +import warnings + +import h11 + +from ..base import Request, Response +from ..exceptions import ( + ConnectTimeoutError, + NewConnectionError, + SubjectAltNameWarning, + SystemTimeWarning, + BadVersionError, + FailedTunnelError, + InvalidBodyError, + ProtocolError, +) +from ..packages import six +from ..util import ssl_ as ssl_util +from .._backends import SyncBackend +from .._backends._common import LoopAbort + +try: + import ssl +except ImportError: + ssl = None +# When updating RECENT_DATE, move it to +# within two years of the current date, and no +# earlier than 6 months ago. +RECENT_DATE = datetime.date(2016, 1, 1) +_SUPPORTED_VERSIONS = frozenset([b'1.0', b'1.1']) +# A sentinel object returned when some syscalls return EAGAIN. +_EAGAIN = object() + + +def _headers_to_native_string(headers): + """ + A temporary shim to convert received headers to native strings, to match + the behaviour of httplib. We will reconsider this later in the process. + """ + # TODO: revisit. + # This works because fundamentally we know that all headers coming from + # h11 are bytes, so if they aren't of type `str` then we must be on Python + # 3 and need to decode the headers using Latin1. + for n, v in headers: + if not isinstance(n, str): + n = n.decode('latin1') + if not isinstance(v, str): + v = v.decode('latin1') + yield (n, v) + + +def _stringify_headers(headers): + """ + A generator that transforms headers so they're suitable for sending by h11. + """ + # TODO: revisit + for name, value in headers: + if isinstance(name, six.text_type): + name = name.encode('ascii') + if isinstance(value, six.text_type): + value = value.encode('latin-1') + elif isinstance(value, int): + value = str(value).encode('ascii') + yield (name, value) + + +def _read_readable(readable): + # TODO: reconsider this block size + blocksize = 8192 + while True: + datablock = readable.read(blocksize) + if not datablock: + break + + yield datablock + + + + +# XX this should return an async iterator +def _make_body_iterable(body): + """ + This function turns all possible body types that urllib3 supports into an + iterable of bytes. The goal is to expose a uniform structure to request + bodies so that they all appear to be identical to the low-level code. + + The basic logic here is: + - byte strings are turned into single-element lists + - readables are wrapped in an iterable that repeatedly calls read until + nothing is returned anymore + - other iterables are used directly + - anything else is not acceptable + + In particular, note that we do not support *text* data of any kind. This + is deliberate: users must make choices about the encoding of the data they + use. + """ + if body is None: + return [] + + elif isinstance(body, six.binary_type): + return [body] + + elif hasattr(body, "read"): + return _read_readable(body) + + elif isinstance(body, collections.Iterable) and not isinstance( + body, six.text_type + ): + return body + + else: + raise InvalidBodyError("Unacceptable body type: %s" % type(body)) + + + + +# XX this should return an async iterator +def _request_bytes_iterable(request, state_machine): + """ + An iterable that serialises a set of bytes for the body. + """ + h11_request = h11.Request( + method=request.method, + target=request.target, + headers=_stringify_headers(request.headers.items()), + ) + yield state_machine.send(h11_request) + + for chunk in _make_body_iterable(request.body): + yield state_machine.send(h11.Data(data=chunk)) + + yield state_machine.send(h11.EndOfMessage()) + + +def _response_from_h11(h11_response, body_object): + """ + Given a h11 Response object, build a urllib3 response object and return it. + """ + if h11_response.http_version not in _SUPPORTED_VERSIONS: + raise BadVersionError(h11_response.http_version) + + version = b'HTTP/' + h11_response.http_version + our_response = Response( + status_code=h11_response.status_code, + headers=_headers_to_native_string(h11_response.headers), + body=body_object, + version=version, + ) + return our_response + + +def _build_tunnel_request(host, port, headers): + """ + Builds a urllib3 Request object that is set up correctly to request a proxy + to establish a TCP tunnel to the remote host. + """ + target = "%s:%d" % (host, port) + if not isinstance(target, bytes): + target = target.encode('latin1') + tunnel_request = Request(method=b"CONNECT", target=target, headers=headers) + tunnel_request.add_host(host=host, port=port, scheme='http') + return tunnel_request + + +async def _start_http_request(request, state_machine, conn): + """ + Send the request using the given state machine and connection, wait + for the response headers, and return them. + + If we get response headers early, then we stop sending and return + immediately, poisoning the state machine along the way so that we know + it can't be re-used. + + This is a standalone function because we use it both to set up both + CONNECT requests and real requests. + """ + # Before we begin, confirm that the state machine is ok. + if ( + state_machine.our_state is not h11.IDLE or + state_machine.their_state is not h11.IDLE + ): + raise ProtocolError("Invalid internal state transition") + + request_bytes_iterable = _request_bytes_iterable(request, state_machine) + # Hack around Python 2 lack of nonlocal + context = {'send_aborted': True, 'h11_response': None} + + async def next_bytes_to_send(): + try: + return next(request_bytes_iterable) + + except StopIteration: + # We successfully sent the whole body! + context['send_aborted'] = False + return None + + def consume_bytes(data): + state_machine.receive_data(data) + while True: + event = state_machine.next_event() + if event is h11.NEED_DATA: + break + + elif isinstance(event, h11.InformationalResponse): + # Ignore 1xx responses + continue + + elif isinstance(event, h11.Response): + # We have our response! Save it and get out of here. + context['h11_response'] = event + raise LoopAbort + + else: + # Can't happen + raise RuntimeError("Unexpected h11 event {}".format(event)) + + await conn.send_and_receive_for_a_while(next_bytes_to_send, consume_bytes) + assert context['h11_response'] is not None + if context['send_aborted']: + # Our state machine thinks we sent a bunch of data... but maybe we + # didn't! Maybe our send got cancelled while we were only half-way + # through sending the last chunk, and then h11 thinks we sent a + # complete request and we actually didn't. Then h11 might think we can + # re-use this connection, even though we can't. So record this in + # h11's state machine. + # XX need to implement this in h11 + # state_machine.poison() + # XX kluge for now + state_machine._cstate.process_error(state_machine.our_role) + return context['h11_response'] + + +async def _read_until_event(state_machine, conn): + """ + A loop that keeps issuing reads and feeding the data into h11 and + checking whether h11 has an event for us. The moment there is an event + other than h11.NEED_DATA, this function returns that event. + """ + while True: + event = state_machine.next_event() + if event is not h11.NEED_DATA: + return event + + state_machine.receive_data(await conn.receive_some()) + + +_DEFAULT_SOCKET_OPTIONS = object() + + +class HTTP1Connection(object): + """ + A wrapper around a single HTTP/1.1 connection. + + This wrapper manages connection state, ensuring that connections are + appropriately managed throughout the lifetime of a HTTP transaction. In + particular, this object understands the conditions in which connections + should be torn down, and also manages sending data and handling early + responses. + + This object can be iterated over to return the response body. When iterated + over it will return all of the data that is currently buffered, and if no + data is buffered it will issue one read syscall and return all of that + data. Buffering of response data must happen at a higher layer. + """ + # : Disable Nagle's algorithm by default. + #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]`` + default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)] + + def __init__( + self, + host, + port, + backend=None, + socket_options=_DEFAULT_SOCKET_OPTIONS, + source_address=None, + tunnel_host=None, + tunnel_port=None, + tunnel_headers=None, + ): + self.is_verified = False + self._backend = backend or SyncBackend() + self._host = host + self._port = port + self._socket_options = ( + socket_options if socket_options is not _DEFAULT_SOCKET_OPTIONS else self.default_socket_options + ) + self._source_address = source_address + self._tunnel_host = tunnel_host + self._tunnel_port = tunnel_port + self._tunnel_headers = tunnel_headers + self._sock = None + self._state_machine = h11.Connection(our_role=h11.CLIENT) + + async def _wrap_socket( + self, conn, ssl_context, fingerprint, assert_hostname + ): + """ + Handles extra logic to wrap the socket in TLS magic. + """ + is_time_off = datetime.date.today() < RECENT_DATE + if is_time_off: + warnings.warn( + ( + 'System time is way off (before {0}). This will probably ' + 'lead to SSL verification errors' + ).format( + RECENT_DATE + ), + SystemTimeWarning, + ) + # XX need to know whether this is the proxy or the final host that + # we just did a handshake with! + check_host = assert_hostname or self._tunnel_host or self._host + # Stripping trailing dots from the hostname is important because + # they indicate that this host is an absolute name (for DNS + # lookup), but are irrelevant to SSL hostname matching and in fact + # will break it. + check_host = check_host.rstrip(".") + conn = await conn.start_tls(check_host, ssl_context) + if fingerprint: + ssl_util.assert_fingerprint( + conn.getpeercert(binary_form=True), fingerprint + ) + elif ( + ssl_context.verify_mode != ssl.CERT_NONE and + assert_hostname is not False + ): + cert = conn.getpeercert() + if not cert.get('subjectAltName', ()): + warnings.warn( + ( + 'Certificate for {0} has no `subjectAltName`, falling ' + 'back to check for a `commonName` for now. This ' + 'feature is being removed by major browsers and ' + 'deprecated by RFC 2818. (See ' + 'https://github.com/shazow/urllib3/issues/497 for ' + 'details.)'.format(self._host) + ), + SubjectAltNameWarning, + ) + ssl_util.match_hostname(cert, check_host) + self.is_verified = ( + ssl_context.verify_mode == ssl.CERT_REQUIRED and + (assert_hostname is not False or fingerprint) + ) + return conn + + async def send_request(self, request, read_timeout): + """ + Given a Request object, performs the logic required to get a response. + """ + h11_response = await _start_http_request( + request, self._state_machine, self._sock + ) + return _response_from_h11(h11_response, self) + + async def _tunnel(self, conn): + """ + This method establishes a CONNECT tunnel shortly after connection. + """ + # Basic sanity check that _tunnel is only called at appropriate times. + assert self._state_machine.our_state is h11.IDLE + tunnel_request = _build_tunnel_request( + self._tunnel_host, self._tunnel_port, self._tunnel_headers + ) + tunnel_state_machine = h11.Connection(our_role=h11.CLIENT) + h11_response = await _start_http_request( + tunnel_request, tunnel_state_machine, conn + ) + # XX this is wrong -- 'self' here will try to iterate using + # self._state_machine, not tunnel_state_machine. Also, we need to + # think about how this failure case interacts with the pool's + # connection lifecycle management. + tunnel_response = _response_from_h11(h11_response, self) + if h11_response.status_code != 200: + conn.forceful_close() + raise FailedTunnelError( + "Unable to establish CONNECT tunnel", tunnel_response + ) + + async def connect( + self, + ssl_context=None, + fingerprint=None, + assert_hostname=None, + connect_timeout=None, + ): + """ + Connect this socket to the server, applying the source address, any + relevant socket options, and the relevant connection timeout. + """ + if self._sock is not None: + # We're already connected, move on. + self._sock.set_readable_watch_state(False) + return + + extra_kw = {} + if self._source_address: + extra_kw['source_address'] = self._source_address + if self._socket_options: + extra_kw['socket_options'] = self._socket_options + # XX pass connect_timeout to backend + # This was factored out into a separate function to allow overriding + # by subclasses, but in the backend approach the way to to this is to + # provide a custom backend. (Composition >> inheritance.) + try: + conn = await self._backend.connect( + self._host, self._port, **extra_kw + ) + # XX these two error handling blocks needs to be re-done in a + # backend-agnostic way + except socket.timeout: + raise ConnectTimeoutError( + self, + "Connection to %s timed out. (connect timeout=%s)" % + (self._host, connect_timeout), + ) + + except socket.error as e: + raise NewConnectionError( + self, "Failed to establish a new connection: %s" % e + ) + + if ssl_context is not None: + if self._tunnel_host is not None: + self._tunnel(conn) + conn = await self._wrap_socket( + conn, ssl_context, fingerprint, assert_hostname + ) + # XX We should pick one of these names and use it consistently... + self._sock = conn + + def close(self): + """ + Close this connection. + """ + if self._sock is not None: + # Make sure self._sock is None even if closing raises an exception + sock, self._sock = self._sock, None + sock.forceful_close() + + def is_dropped(self): + """ + Returns True if the connection is closed: returns False otherwise. This + includes closures that do not mark the FD as closed, such as when the + remote peer has sent EOF but we haven't read it yet. + + Pre-condition: _reset must have been called. + """ + if self._sock is None: + return True + + # We check for droppedness by checking the socket for readability. If + # it's not readable, it's not dropped. If it is readable, then we + # assume that the thing we'd read from the socket is EOF. It might not + # be, but if it's not then the server has busted its HTTP/1.1 framing + # and so we want to drop the connection anyway. + return self._sock.is_readable() + + def _reset(self): + """ + Called once we hit EndOfMessage, and checks whether we can re-use this + state machine and connection or not, and if not, closes the socket and + state machine. + """ + try: + self._state_machine.start_next_cycle() + except h11.LocalProtocolError: + # Not re-usable + self.close() + else: + # This connection can be returned to the connection pool, and + # eventually we'll take it out again and want to know if it's been + # dropped. + self._sock.set_readable_watch_state(True) + + @property + def complete(self): + """ + XX what is this supposed to do? check if the response has been fully + iterated over? check for that + the connection being reusable? + """ + our_state = self._state_machine.our_state + their_state = self._state_machine.their_state + return (our_state is h11.IDLE and their_state is h11.IDLE) + + def __aiter__(self): + return self + + def next(self): # Needed for Python 2 as __anext__ becomes __next__ + return self.__next__() + + async def __anext__(self): + """ + Iterate over the body bytes of the response until end of message. + """ + event = await _read_until_event(self._state_machine, self._sock) + if isinstance(event, h11.Data): + return bytes(event.data) + + elif isinstance(event, h11.EndOfMessage): + self._reset() + raise StopAsyncIteration + + else: + # can't happen + raise RuntimeError("Unexpected h11 event {}".format(event)) diff --git a/requests3/core/_http/_async/connectionpool.py b/requests3/core/_http/_async/connectionpool.py new file mode 100644 index 00000000..3c829c3c --- /dev/null +++ b/requests3/core/_http/_async/connectionpool.py @@ -0,0 +1,891 @@ +from __future__ import absolute_import +import errno +import logging +import sys +import warnings + +from socket import error as SocketError, timeout as SocketTimeout +import socket + +import h11 + + +from ..base import Request, DEFAULT_PORTS +from ..exceptions import ( + ClosedPoolError, + ProtocolError, + EmptyPoolError, + LocationValueError, + MaxRetryError, + ProxyError, + ReadTimeoutError, + SSLError, + TimeoutError, + InsecureRequestWarning, + NewConnectionError, +) +from ..packages.ssl_match_hostname import CertificateError +from ..packages import six +from ..packages.six.moves import queue +from ..request import RequestMethods +from .response import HTTPResponse +from .connection import HTTP1Connection + +from ..util.connection import is_connection_dropped +from ..util.request import set_file_position +from ..util.retry import Retry +from ..util.ssl_ import ( + create_urllib3_context, + merge_context_settings, + resolve_ssl_version, + resolve_cert_reqs, + BaseSSLError, +) +from ..util.timeout import Timeout +from ..util.url import get_host, Url + +try: + import ssl +except ImportError: + ssl = None +if six.PY2: + # Queue is imported for side effects on MS Windows + import Queue as _unused_module_Queue # noqa: F401 +xrange = six.moves.xrange +log = logging.getLogger(__name__) +_Default = object() + + +def _add_transport_headers(headers): + """ + Adds the transport framing headers, if needed. Naturally, this method + cannot add a content-length header, so if there is no content-length header + then it will add Transfer-Encoding: chunked instead. Should only be called + if there is a body to upload. + + This should be a bit smarter: in particular, it should allow for bad or + unexpected versions of these headers, particularly transfer-encoding. + """ + transfer_headers = ('content-length', 'transfer-encoding') + for header_name in headers: + if header_name.lower() in transfer_headers: + return + + headers['transfer-encoding'] = 'chunked' + + +def _build_context( + context, keyfile, certfile, cert_reqs, ca_certs, ca_cert_dir, ssl_version +): + """ + Creates a urllib3 context suitable for a given request based on a + collection of possible properties of that context. + """ + if context is None: + context = create_urllib3_context( + ssl_version=resolve_ssl_version(ssl_version), + cert_reqs=resolve_cert_reqs(cert_reqs), + ) + context = merge_context_settings( + context, + keyfile=keyfile, + certfile=certfile, + cert_reqs=cert_reqs, + ca_certs=ca_certs, + ca_cert_dir=ca_cert_dir, + ) + return context + + + + +# Pool objects +class ConnectionPool(object): + """ + Base class for all connection pools, such as + :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`. + """ + scheme = None + QueueCls = queue.LifoQueue + + def __init__(self, host, port=None): + if not host: + raise LocationValueError("No host specified.") + + self.host = _ipv6_host(host).lower() + self.port = port + + def __str__(self): + return '%s(host=%r, port=%r)' % ( + type(self).__name__, self.host, self.port + ) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + # Return False to re-raise any potential exceptions + return False + + def close(self): + """ + Close all pooled connections and disable the pool. + """ + pass + + +# This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 +_blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK]) + + +class HTTPConnectionPool(ConnectionPool, RequestMethods): + """ + Thread-safe connection pool for one host. + + :param host: + Host used for this HTTP Connection (e.g. "localhost"), passed into + :class:`httplib.HTTPConnection`. + + :param port: + Port used for this HTTP Connection (None is equivalent to 80), passed + into :class:`httplib.HTTPConnection`. + + :param strict: + Causes BadStatusLine to be raised if the status line can't be parsed + as a valid HTTP/1.0 or 1.1 status line, passed into + :class:`httplib.HTTPConnection`. + + .. note:: + Only works in Python 2. This parameter is ignored in Python 3. + + :param timeout: + Socket timeout in seconds for each individual connection. This can + be a float or integer, which sets the timeout for the HTTP request, + or an instance of :class:`urllib3.util.Timeout` which gives you more + fine-grained control over request timeouts. After the constructor has + been parsed, this is always a `urllib3.util.Timeout` object. + + :param maxsize: + Number of connections to save that can be reused. More than 1 is useful + in multithreaded situations. If ``block`` is set to False, more + connections will be created but they will not be saved once they've + been used. + + :param block: + If set to True, no more than ``maxsize`` connections will be used at + a time. When no free connections are available, the call will block + until a connection has been released. This is a useful side effect for + particular multithreaded situations where one does not want to use more + than maxsize connections per host to prevent flooding. + + :param headers: + Headers to include with all requests, unless other headers are given + explicitly. + + :param retries: + Retry configuration to use by default with requests in this pool. + + :param _proxy: + Parsed proxy URL, should not be used directly, instead, see + :class:`urllib3.connectionpool.ProxyManager`" + + :param _proxy_headers: + A dictionary with proxy headers, should not be used directly, + instead, see :class:`urllib3.connectionpool.ProxyManager`" + + :param \\**conn_kw: + Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`, + :class:`urllib3.connection.HTTPSConnection` instances. + """ + scheme = 'http' + ConnectionCls = HTTP1Connection + ResponseCls = HTTPResponse + + def __init__( + self, + host, + port=None, + timeout=Timeout.DEFAULT_TIMEOUT, + maxsize=1, + block=False, + headers=None, + retries=None, + _proxy=None, + _proxy_headers=None, + **conn_kw + ): + ConnectionPool.__init__(self, host, port) + RequestMethods.__init__(self, headers) + if not isinstance(timeout, Timeout): + timeout = Timeout.from_float(timeout) + if retries is None: + retries = Retry.DEFAULT + self.timeout = timeout + self.retries = retries + self.pool = self.QueueCls(maxsize) + self.block = block + self.proxy = _proxy + self.proxy_headers = _proxy_headers or {} + # Fill the queue up so that doing get() on it will block properly + for _ in xrange(maxsize): + self.pool.put(None) + # These are mostly for testing and debugging purposes. + self.num_connections = 0 + self.num_requests = 0 + self.conn_kw = conn_kw + if self.proxy: + # Enable Nagle's algorithm for proxies, to avoid packet fragmentation. + # We cannot know if the user has added default socket options, so we cannot replace the + # list. + self.conn_kw.setdefault('socket_options', []) + + def _new_conn(self): + """ + Return a fresh connection. + """ + self.num_connections += 1 + + # TODO: Huge hack. + for kw in ('strict',): + if kw in self.conn_kw: + self.conn_kw.pop(kw) + + log.debug( + "Starting new HTTP connection (%d): %s:%s", + self.num_connections, + self.host, + self.port or "80", + ) + conn = self.ConnectionCls( + host=self.host, port=self.port, ** self.conn_kw + ) + return conn + + async def _get_conn(self, timeout=None): + """ + Get a connection. Will return a pooled connection if one is available. + + If no connections are available and :prop:`.block` is ``False``, then a + fresh connection is returned. + + :param timeout: + Seconds to wait before giving up and raising + :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and + :prop:`.block` is ``True``. + """ + conn = None + try: + conn = self.pool.get(block=self.block, timeout=timeout) + except AttributeError: # self.pool is None + raise ClosedPoolError(self, "Pool is closed.") + + except queue.Empty: + if self.block: + raise EmptyPoolError( + self, + "Pool reached maximum size and no more " + "connections are allowed.", + ) + + pass # Oh well, we'll create a new connection then + # If this is a persistent connection, check if it got disconnected + if conn and is_connection_dropped(conn): + log.debug("Resetting dropped connection: %s", self.host) + conn.close() + return conn or self._new_conn() + + async def _put_conn(self, conn): + """ + Put a connection back into the pool. + + :param conn: + Connection object for the current host and port as returned by + :meth:`._new_conn` or :meth:`._get_conn`. + + If the pool is already full, the connection is closed and discarded + because we exceeded maxsize. If connections are discarded frequently, + then maxsize should be increased. + + If the pool is closed, then the connection will be closed and discarded. + """ + try: + self.pool.put(conn, block=False) + return # Everything is dandy, done. + + except AttributeError: + # self.pool is None. + pass + except queue.Full: + # This should never happen if self.block == True + log.warning( + "Connection pool is full, discarding connection: %s", self.host + ) + # Connection never got put back into the pool, close it. + if conn: + conn.close() + + async def _start_conn(self, conn, connect_timeout): + """ + Called right before a request is made, after the socket is created. + """ + await conn.connect(connect_timeout=connect_timeout) + + def _get_timeout(self, timeout): + """ Helper that always returns a :class:`urllib3.util.Timeout` """ + if timeout is _Default: + return self.timeout.clone() + + if isinstance(timeout, Timeout): + return timeout.clone() + + else: + # User passed us an int/float. This is for backwards compatibility, + # can be removed later + return Timeout.from_float(timeout) + + def _raise_timeout(self, err, url, timeout_value): + """Is the error actually a timeout? Will raise a ReadTimeout or pass""" + if isinstance(err, SocketTimeout): + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % timeout_value + ) + + # See the above comment about EAGAIN in Python 3. In Python 2 we have + # to specifically catch it and throw the timeout error + if hasattr(err, 'errno') and err.errno in _blocking_errnos: + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % timeout_value + ) + + # Catch possible read timeouts thrown as SSL errors. If not the + # case, rethrow the original. We need to do this because of: + # http://bugs.python.org/issue10272 + # TODO: Can we remove this? + if 'timed out' in str(err) or 'did not complete (read)' in str( + err + ): # Python 2.6 + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % timeout_value + ) + + async def _make_request( + self, conn, method, url, timeout=_Default, body=None, headers=None + ): + """ + Perform a request on a given urllib connection object taken from our + pool. + + :param conn: + a connection from one of our connection pools + + :param timeout: + Socket timeout in seconds for the request. This can be a + float or integer, which will set the same timeout value for + the socket connect and the socket read, or an instance of + :class:`urllib3.util.Timeout`, which gives you more fine-grained + control over your timeouts. + """ + self.num_requests += 1 + timeout_obj = self._get_timeout(timeout) + timeout_obj.start_connect() + # Trigger any extra validation we need to do. + try: + await self._start_conn(conn, timeout_obj.connect_timeout) + except (SocketTimeout, BaseSSLError) as e: + # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout. + self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) + raise + + # TODO: We need to encapsulate our proxy logic in here somewhere. + request = Request( + method=method, target=url, headers=headers, body=body + ) + host = self.host + port = self.port + scheme = self.scheme + request.add_host(host, port, scheme) + # Reset the timeout for the recv() on the socket + read_timeout = timeout_obj.read_timeout + # In Python 3 socket.py will catch EAGAIN and return None when you + # try and read into the file pointer created by http.client, which + # instead raises a BadStatusLine exception. Instead of catching + # the exception and assuming all BadStatusLine exceptions are read + # timeouts, check for a zero timeout before making the request. + if read_timeout == 0: + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % read_timeout + ) + + if read_timeout is Timeout.DEFAULT_TIMEOUT: + read_timeout = socket.getdefaulttimeout() + # Receive the response from the server + try: + response = await conn.send_request( + request, read_timeout=read_timeout + ) + except (SocketTimeout, BaseSSLError, SocketError) as e: + self._raise_timeout(err=e, url=url, timeout_value=read_timeout) + raise + + # AppEngine doesn't have a version attr. + http_version = getattr(conn, '_http_vsn_str', 'HTTP/?') + log.debug( + "%s://%s:%s \"%s %s %s\" %s", + self.scheme, + self.host, + self.port, + method, + url, + http_version, + response.status_code, + ) + return response + + def _absolute_url(self, path): + return Url( + scheme=self.scheme, host=self.host, port=self.port, path=path + ).url + + def close(self): + """ + Close all pooled connections and disable the pool. + """ + if self.pool is None: + return + + # Disable access to the pool + old_pool, self.pool = self.pool, None + try: + while True: + conn = old_pool.get(block=False) + if conn: + conn.close() + except queue.Empty: + pass # Done. + + def is_same_host(self, url): + """ + Check if the given ``url`` is a member of the same host as this + connection pool. + """ + if url.startswith('/'): + return True + + # TODO: Add optional support for socket.gethostbyname checking. + scheme, host, port = get_host(url) + host = _ipv6_host(host).lower() + # Use explicit default port for comparison when none is given + if self.port and not port: + port = DEFAULT_PORTS.get(scheme) + elif not self.port and port == DEFAULT_PORTS.get(scheme): + port = None + return (scheme, host, port) == (self.scheme, self.host, self.port) + + async def urlopen( + self, + method, + url, + body=None, + headers=None, + retries=None, + timeout=_Default, + pool_timeout=None, + body_pos=None, + **response_kw + ): + """ + Get a connection from the pool and perform an HTTP request. This is the + lowest level call for making a request, so you'll need to specify all + the raw details. + + .. note:: + + More commonly, it's appropriate to use a convenience method provided + by :class:`.RequestMethods`, such as :meth:`request`. + + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param body: + Data to send in the request body (useful for creating + POST requests, see HTTPConnectionPool.post_url for + more convenience). + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. If None, pool headers are used. If provided, + these headers completely replace any pool-specific headers. + + :param retries: + Configure the number of retries to allow before raising a + :class:`~urllib3.exceptions.MaxRetryError` exception. + + Pass ``None`` to retry until you receive a response. Pass a + :class:`~urllib3.util.retry.Retry` object for fine-grained control + over different types of retries. + Pass an integer number to retry connection errors that many times, + but no other types of errors. Pass zero to never retry. + + If ``False``, then retries are disabled and any exception is raised + immediately. Also, instead of raising a MaxRetryError on redirects, + the redirect response will be returned. + + :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. + + :param timeout: + If specified, overrides the default timeout for this one + request. It may be a float (in seconds) or an instance of + :class:`urllib3.util.Timeout`. + + :param pool_timeout: + If set and the pool is set to block=True, then this method will + block for ``pool_timeout`` seconds and raise EmptyPoolError if no + connection is available within the time period. + + :param int body_pos: + Position to seek to in file-like body in the event of a retry or + redirect. Typically this won't need to be set because urllib3 will + auto-populate the value when needed. + + :param \\**response_kw: + Additional parameters are passed to + :meth:`urllib3.response.HTTPResponse.from_httplib` + """ + if headers is None: + headers = self.headers + if not isinstance(retries, Retry): + retries = Retry.from_int( + retries, default=self.retries, redirect=False + ) + conn = None + # Track whether `conn` needs to be released before + # returning/raising/recursing. + release_this_conn = False + # Merge the proxy headers. Only do this in HTTP. We have to copy the + # headers dict so we can safely change it without those changes being + # reflected in anyone else's copy. + if self.scheme == 'http': + headers = headers.copy() + headers.update(self.proxy_headers) + # Must keep the exception bound to a separate variable or else Python 3 + # complains about UnboundLocalError. + err = None + # Keep track of whether we cleanly exited the except block. This + # ensures we do proper cleanup in finally. + clean_exit = False + # Rewind body position, if needed. Record current position + # for future rewinds in the event of a redirect/retry. + body_pos = set_file_position(body, body_pos) + if body is not None: + _add_transport_headers(headers) + try: + # Request a connection from the queue. + timeout_obj = self._get_timeout(timeout) + conn = await self._get_conn(timeout=pool_timeout) + conn.timeout = timeout_obj.connect_timeout + # Make the request on the base connection object. + base_response = await self._make_request( + conn, + method, + url, + timeout=timeout_obj, + body=body, + headers=headers, + ) + # Pass method to Response for length checking + response_kw['request_method'] = method + # Import httplib's response into our own wrapper object + response = self.ResponseCls.from_base( + base_response, pool=self, retries=retries, **response_kw + ) + # Everything went great! + clean_exit = True + except queue.Empty: + # Timed out by queue. + raise EmptyPoolError(self, "No pool connections are available.") + + except ( + TimeoutError, + SocketError, + ProtocolError, + h11.ProtocolError, + BaseSSLError, + SSLError, + CertificateError, + ) as e: + # Discard the connection for these exceptions. It will be + # replaced during the next _get_conn() call. + clean_exit = False + if isinstance(e, (BaseSSLError, CertificateError)): + e = SSLError(e) + elif isinstance( + e, (SocketError, NewConnectionError) + ) and self.proxy: + e = ProxyError('Cannot connect to proxy.', e) + elif isinstance(e, (SocketError, h11.ProtocolError)): + e = ProtocolError('Connection aborted.', e) + retries = retries.increment( + method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] + ) + retries.sleep() + # Keep track of the error for the retry warning. + err = e + finally: + if not clean_exit: + # We hit some kind of exception, handled or otherwise. We need + # to throw the connection away unless explicitly told not to. + # Close the connection, set the variable to None, and make sure + # we put the None back in the pool to avoid leaking it. + conn = conn and conn.close() + release_this_conn = True + if release_this_conn: + # Put the connection back to be reused. If the connection is + # expired then it will be None, which will get replaced with a + # fresh connection during _get_conn. + await self._put_conn(conn) + if not conn: + # Try again + log.warning( + "Retrying (%r) after connection " "broken by '%r': %s", + retries, + err, + url, + ) + return await self.urlopen( + method, + url, + body, + headers, + retries, + timeout=timeout, + pool_timeout=pool_timeout, + body_pos=body_pos, + **response_kw + ) + + def drain_and_release_conn(response): + try: + # discard any remaining response body, the connection will be + # released back to the pool once the entire response is read + response.read() + except ( + TimeoutError, + SocketError, + ProtocolError, + BaseSSLError, + SSLError, + ) as e: + pass + + # Check if we should retry the HTTP response. + has_retry_after = bool(response.getheader('Retry-After')) + if retries.is_retry(method, response.status, has_retry_after): + try: + retries = retries.increment( + method, url, response=response, _pool=self + ) + except MaxRetryError: + if retries.raise_on_status: + # Drain and release the connection for this response, since + # we're not returning it to be released manually. + drain_and_release_conn(response) + raise + + return response + + # drain and return the connection to the pool before recursing + drain_and_release_conn(response) + retries.sleep(response) + log.debug("Retry: %s", url) + return await self.urlopen( + method, + url, + body, + headers, + retries=retries, + timeout=timeout, + pool_timeout=pool_timeout, + body_pos=body_pos, + **response_kw + ) + + return response + + +class HTTPSConnectionPool(HTTPConnectionPool): + """ + Same as :class:`.HTTPConnectionPool`, but HTTPS. + + When Python is compiled with the :mod:`ssl` module, then + :class:`.VerifiedHTTPSConnection` is used, which *can* verify certificates, + instead of :class:`.HTTPSConnection`. + + :class:`.VerifiedHTTPSConnection` uses one of ``assert_fingerprint``, + ``assert_hostname`` and ``host`` in this order to verify connections. + If ``assert_hostname`` is False, no verification is done. + + The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``, + ``ca_cert_dir``, and ``ssl_version`` are only used if :mod:`ssl` is + available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade + the connection socket into an SSL socket. + """ + scheme = 'https' + + def __init__( + self, + host, + port=None, + timeout=Timeout.DEFAULT_TIMEOUT, + maxsize=1, + block=False, + headers=None, + retries=None, + _proxy=None, + _proxy_headers=None, + key_file=None, + cert_file=None, + cert_reqs=None, + ca_certs=None, + ssl_version=None, + assert_hostname=None, + assert_fingerprint=None, + ca_cert_dir=None, + ssl_context=None, + **conn_kw + ): + HTTPConnectionPool.__init__( + self, + host, + port, + timeout, + maxsize, + block, + headers, + retries, + _proxy, + _proxy_headers, + **conn_kw + ) + if ssl is None: + raise SSLError("SSL module is not available") + + if ca_certs and cert_reqs is None: + cert_reqs = 'CERT_REQUIRED' + self.ssl_context = _build_context( + ssl_context, + keyfile=key_file, + certfile=cert_file, + cert_reqs=cert_reqs, + ca_certs=ca_certs, + ca_cert_dir=ca_cert_dir, + ssl_version=ssl_version, + ) + self.assert_hostname = assert_hostname + self.assert_fingerprint = assert_fingerprint + + def _new_conn(self): + """ + Return a fresh connection. + """ + self.num_connections += 1 + log.debug( + "Starting new HTTPS connection (%d): %s:%s", + self.num_connections, + self.host, + self.port or "443", + ) + actual_host = self.host + actual_port = self.port + tunnel_host = None + tunnel_port = None + tunnel_headers = None + if self.proxy is not None: + actual_host = self.proxy.host + actual_port = self.proxy.port + tunnel_host = self.host + tunnel_port = self.port + tunnel_headers = self.proxy_headers + + # TODO: Huge hack. + for kw in ('strict', 'redirect'): + if kw in self.conn_kw: + self.conn_kw.pop(kw) + + conn = self.ConnectionCls( + host=actual_host, + port=actual_port, + tunnel_host=tunnel_host, + tunnel_port=tunnel_port, + tunnel_headers=tunnel_headers, + ** self.conn_kw + ) + return conn + + async def _start_conn(self, conn, connect_timeout): + """ + Called right before a request is made, after the socket is created. + """ + await conn.connect( + ssl_context=self.ssl_context, + fingerprint=self.assert_fingerprint, + assert_hostname=self.assert_hostname, + connect_timeout=connect_timeout, + ) + if not conn.is_verified: + warnings.warn( + ( + 'Unverified HTTPS request is being made. ' + 'Adding certificate verification is strongly advised. See: ' + 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' + '#ssl-warnings' + ), + InsecureRequestWarning, + ) + + +def connection_from_url(url, **kw): + """ + Given a url, return an :class:`.ConnectionPool` instance of its host. + + This is a shortcut for not having to parse out the scheme, host, and port + of the url before creating an :class:`.ConnectionPool` instance. + + :param url: + Absolute URL string that must include the scheme. Port is optional. + + :param \\**kw: + Passes additional parameters to the constructor of the appropriate + :class:`.ConnectionPool`. Useful for specifying things like + timeout, maxsize, headers, etc. + + Example:: + + >>> conn = connection_from_url('http://google.com/') + >>> r = conn.request('GET', '/') + """ + scheme, host, port = get_host(url) + port = port or DEFAULT_PORTS.get(scheme, 80) + if scheme == 'https': + return HTTPSConnectionPool(host, port=port, **kw) + + else: + return HTTPConnectionPool(host, port=port, **kw) + + +def _ipv6_host(host): + """ + Process IPv6 address literals + """ + # httplib doesn't like it when we include brackets in IPv6 addresses + # Specifically, if we include brackets but also pass the port then + # httplib crazily doubles up the square brackets on the Host header. + # Instead, we need to make sure we never pass ``None`` as the port. + # However, for backward compatibility reasons we can't actually + # *assert* that. See http://bugs.python.org/issue28539 + # + # Also if an IPv6 address literal has a zone identifier, the + # percent sign might be URIencoded, convert it back into ASCII + if host.startswith('[') and host.endswith(']'): + host = host.replace('%25', '%').strip('[]') + return host diff --git a/requests3/core/_http/_async/poolmanager.py b/requests3/core/_http/_async/poolmanager.py new file mode 100644 index 00000000..0645a0f5 --- /dev/null +++ b/requests3/core/_http/_async/poolmanager.py @@ -0,0 +1,446 @@ +from __future__ import absolute_import +import collections +import functools +import logging + +from .._collections import RecentlyUsedContainer +from ..base import DEFAULT_PORTS +from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool +from ..exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown +from ..packages.six.moves.urllib.parse import urljoin +from ..request import RequestMethods +from ..util.url import parse_url +from ..util.request import set_file_position +from ..util.retry import Retry + +__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] +log = logging.getLogger(__name__) +SSL_KEYWORDS = ( + 'key_file', + 'cert_file', + 'cert_reqs', + 'ca_certs', + 'ssl_version', + 'ca_cert_dir', + 'ssl_context', +) +# All known keyword arguments that could be provided to the pool manager, its +# pools, or the underlying connections. This is used to construct a pool key. +_key_fields = ( + 'key_scheme', # str + 'key_host', # str + 'key_strict', + 'key_port', # int + 'key_timeout', # int or float or Timeout + 'key_retries', # int or Retry + 'key_block', # bool + 'key_source_address', # str + 'key_key_file', # str + 'key_cert_file', # str + 'key_cert_reqs', # str + 'key_ca_certs', # str + 'key_ssl_version', # str + 'key_ca_cert_dir', # str + 'key_ssl_context', # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext + 'key_maxsize', # int + 'key_headers', # dict + 'key__proxy', # parsed proxy url + 'key__proxy_headers', # dict + 'key_socket_options', # list of (level (int), optname (int), value (int or str)) tuples + 'key__socks_options', # dict + 'key_assert_hostname', # bool or string + 'key_assert_fingerprint', # str +) +# : The namedtuple class used to construct keys for the connection pool. +#: All custom key schemes should include the fields in this key at a minimum. +PoolKey = collections.namedtuple('PoolKey', _key_fields) + + +def _default_key_normalizer(key_class, request_context): + """ + Create a pool key out of a request context dictionary. + + According to RFC 3986, both the scheme and host are case-insensitive. + Therefore, this function normalizes both before constructing the pool + key for an HTTPS request. If you wish to change this behaviour, provide + alternate callables to ``key_fn_by_scheme``. + + :param key_class: + The class to use when constructing the key. This should be a namedtuple + with the ``scheme`` and ``host`` keys at a minimum. + :type key_class: namedtuple + :param request_context: + A dictionary-like object that contain the context for a request. + :type request_context: dict + + :return: A namedtuple that can be used as a connection pool key. + :rtype: PoolKey + """ + # Since we mutate the dictionary, make a copy first + context = request_context.copy() + context['scheme'] = context['scheme'].lower() + context['host'] = context['host'].lower() + # These are both dictionaries and need to be transformed into frozensets + for key in ('headers', '_proxy_headers', '_socks_options'): + if key in context and context[key] is not None: + context[key] = frozenset(context[key].items()) + # The socket_options key may be a list and needs to be transformed into a + # tuple. + socket_opts = context.get('socket_options') + if socket_opts is not None: + context['socket_options'] = tuple(socket_opts) + # Map the kwargs to the names in the namedtuple - this is necessary since + # namedtuples can't have fields starting with '_'. + for key in list(context.keys()): + context['key_' + key] = context.pop(key) + # Default to ``None`` for keys missing from the context + for field in key_class._fields: + if field not in context: + context[field] = None + return key_class(**context) + + +# : A dictionary that maps a scheme to a callable that creates a pool key. +#: This can be used to alter the way pool keys are constructed, if desired. +#: Each PoolManager makes a copy of this dictionary so they can be configured +#: globally here, or individually on the instance. +key_fn_by_scheme = { + 'http': functools.partial(_default_key_normalizer, PoolKey), + 'https': functools.partial(_default_key_normalizer, PoolKey), +} +pool_classes_by_scheme = { + 'http': HTTPConnectionPool, 'https': HTTPSConnectionPool +} + + +class PoolManager(RequestMethods): + """ + Allows for arbitrary requests while transparently keeping track of + necessary connection pools for you. + + :param num_pools: + Number of connection pools to cache before discarding the least + recently used pool. + + :param headers: + Headers to include with all requests, unless other headers are given + explicitly. + + :param \\**connection_pool_kw: + Additional parameters are used to create fresh + :class:`urllib3.connectionpool.ConnectionPool` instances. + + Example:: + + >>> manager = PoolManager(num_pools=2) + >>> r = manager.request('GET', 'http://google.com/') + >>> r = manager.request('GET', 'http://google.com/mail') + >>> r = manager.request('GET', 'http://yahoo.com/') + >>> len(manager.pools) + 2 + + """ + proxy = None + + def __init__( + self, num_pools=10, headers=None, backend=None, **connection_pool_kw + ): + RequestMethods.__init__(self, headers) + self.connection_pool_kw = connection_pool_kw + self.pools = RecentlyUsedContainer( + num_pools, dispose_func=lambda p: p.close() + ) + # Locally set the pool classes and keys so other PoolManagers can + # override them. + self.pool_classes_by_scheme = pool_classes_by_scheme + self.key_fn_by_scheme = key_fn_by_scheme.copy() + self.backend = backend + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.clear() + # Return False to re-raise any potential exceptions + return False + + def _new_pool(self, scheme, host, port, request_context=None): + """ + Create a new :class:`ConnectionPool` based on host, port, scheme, and + any additional pool keyword arguments. + + If ``request_context`` is provided, it is provided as keyword arguments + to the pool class used. This method is used to actually create the + connection pools handed out by :meth:`connection_from_url` and + companion methods. It is intended to be overridden for customization. + """ + pool_cls = self.pool_classes_by_scheme[scheme] + if request_context is None: + request_context = self.connection_pool_kw.copy() + # Although the context has everything necessary to create the pool, + # this function has historically only used the scheme, host, and port + # in the positional args. When an API change is acceptable these can + # be removed. + for key in ('scheme', 'host', 'port'): + request_context.pop(key, None) + if scheme == 'http': + for kw in SSL_KEYWORDS: + request_context.pop(kw, None) + return pool_cls(host, port, backend=self.backend, **request_context) + + def clear(self): + """ + Empty our store of pools and direct them all to close. + + This will not affect in-flight connections, but they will not be + re-used after completion. + """ + self.pools.clear() + + def connection_from_host( + self, host, port=None, scheme='http', pool_kwargs=None + ): + """ + Get a :class:`ConnectionPool` based on the host, port, and scheme. + + If ``port`` isn't given, it will be derived from the ``scheme`` using + ``urllib3.connectionpool.port_by_scheme``. If ``pool_kwargs`` is + provided, it is merged with the instance's ``connection_pool_kw`` + variable and used to create the new connection pool, if one is + needed. + """ + if not host: + raise LocationValueError("No host specified.") + + request_context = self._merge_pool_kwargs(pool_kwargs) + request_context['scheme'] = scheme or 'http' + if not port: + port = DEFAULT_PORTS.get(request_context['scheme'].lower(), 80) + request_context['port'] = port + request_context['host'] = host + return self.connection_from_context(request_context) + + def connection_from_context(self, request_context): + """ + Get a :class:`ConnectionPool` based on the request context. + + ``request_context`` must at least contain the ``scheme`` key and its + value must be a key in ``key_fn_by_scheme`` instance variable. + """ + scheme = request_context['scheme'].lower() + pool_key_constructor = self.key_fn_by_scheme[scheme] + pool_key = pool_key_constructor(request_context) + return self.connection_from_pool_key( + pool_key, request_context=request_context + ) + + def connection_from_pool_key(self, pool_key, request_context=None): + """ + Get a :class:`ConnectionPool` based on the provided pool key. + + ``pool_key`` should be a namedtuple that only contains immutable + objects. At a minimum it must have the ``scheme``, ``host``, and + ``port`` fields. + """ + with self.pools.lock: + # If the scheme, host, or port doesn't match existing open + # connections, open a new ConnectionPool. + pool = self.pools.get(pool_key) + if pool: + return pool + + # Make a fresh ConnectionPool of the desired type + scheme = request_context['scheme'] + host = request_context['host'] + port = request_context['port'] + pool = self._new_pool( + scheme, host, port, request_context=request_context + ) + self.pools[pool_key] = pool + return pool + + def connection_from_url(self, url, pool_kwargs=None): + """ + Similar to :func:`urllib3.connectionpool.connection_from_url`. + + If ``pool_kwargs`` is not provided and a new pool needs to be + constructed, ``self.connection_pool_kw`` is used to initialize + the :class:`urllib3.connectionpool.ConnectionPool`. If ``pool_kwargs`` + is provided, it is used instead. Note that if a new pool does not + need to be created for the request, the provided ``pool_kwargs`` are + not used. + """ + u = parse_url(url) + return self.connection_from_host( + u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs + ) + + def _merge_pool_kwargs(self, override): + """ + Merge a dictionary of override values for self.connection_pool_kw. + + This does not modify self.connection_pool_kw and returns a new dict. + Any keys in the override dictionary with a value of ``None`` are + removed from the merged dictionary. + """ + base_pool_kwargs = self.connection_pool_kw.copy() + if override: + for key, value in override.items(): + if value is None: + try: + del base_pool_kwargs[key] + except KeyError: + pass + else: + base_pool_kwargs[key] = value + return base_pool_kwargs + + async def urlopen(self, method, url, redirect=True, **kw): + """ + Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen` + with redirect logic and only sends the request-uri portion of the + ``url``. + + The given ``url`` parameter must be absolute, such that an appropriate + :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it. + """ + u = parse_url(url) + conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme) + # Rewind body position, if needed. Record current position + # for future rewinds in the event of a redirect/retry. + body = kw.get('body') + body_pos = kw.get('body_pos') + kw['body_pos'] = set_file_position(body, body_pos) + if 'headers' not in kw: + kw['headers'] = self.headers + if self.proxy is not None and u.scheme == "http": + response = await conn.urlopen(method, url, **kw) + else: + response = await conn.urlopen(method, u.request_uri, **kw) + redirect_location = redirect and response.get_redirect_location() + if not redirect_location: + return response + + # Support relative URLs for redirecting. + redirect_location = urljoin(url, redirect_location) + # RFC 7231, Section 6.4.4 + if response.status == 303: + method = 'GET' + retries = kw.get('retries') + if not isinstance(retries, Retry): + retries = Retry.from_int(retries, redirect=redirect) + try: + retries = retries.increment( + method, url, response=response, _pool=conn + ) + except MaxRetryError: + if retries.raise_on_redirect: + raise + + return response + + kw['retries'] = retries + kw['redirect'] = redirect + retries.sleep_for_retry(response) + log.info("Redirecting %s -> %s", url, redirect_location) + return self.urlopen(method, redirect_location, **kw) + + +class ProxyManager(PoolManager): + """ + Behaves just like :class:`PoolManager`, but sends all requests through + the defined proxy, using the CONNECT method for HTTPS URLs. + + :param proxy_url: + The URL of the proxy to be used. + + :param proxy_headers: + A dictionary contaning headers that will be sent to the proxy. In case + of HTTP they are being sent with each request, while in the + HTTPS/CONNECT case they are sent only once. Could be used for proxy + authentication. + + Example: + >>> proxy = urllib3.ProxyManager('http://localhost:3128/') + >>> r1 = proxy.request('GET', 'http://google.com/') + >>> r2 = proxy.request('GET', 'http://httpbin.org/') + >>> len(proxy.pools) + 1 + >>> r3 = proxy.request('GET', 'https://httpbin.org/') + >>> r4 = proxy.request('GET', 'https://twitter.com/') + >>> len(proxy.pools) + 3 + + """ + + def __init__( + self, + proxy_url, + num_pools=10, + headers=None, + proxy_headers=None, + **connection_pool_kw + ): + if isinstance(proxy_url, HTTPConnectionPool): + proxy_url = '%s://%s:%i' % ( + proxy_url.scheme, proxy_url.host, proxy_url.port + ) + proxy = parse_url(proxy_url) + if not proxy.port: + port = DEFAULT_PORTS.get(proxy.scheme, 80) + proxy = proxy._replace(port=port) + if proxy.scheme not in ("http", "https"): + raise ProxySchemeUnknown(proxy.scheme) + + self.proxy = proxy + self.proxy_headers = proxy_headers or {} + connection_pool_kw['_proxy'] = self.proxy + connection_pool_kw['_proxy_headers'] = self.proxy_headers + super(ProxyManager, self).__init__( + num_pools, headers, **connection_pool_kw + ) + + def connection_from_host( + self, host, port=None, scheme='http', pool_kwargs=None + ): + if scheme == "https": + return super(ProxyManager, self).connection_from_host( + host, port, scheme, pool_kwargs=pool_kwargs + ) + + return super(ProxyManager, self).connection_from_host( + self.proxy.host, + self.proxy.port, + self.proxy.scheme, + pool_kwargs=pool_kwargs, + ) + + def _set_proxy_headers(self, url, headers=None): + """ + Sets headers needed by proxies: specifically, the Accept and Host + headers. Only sets headers not provided by the user. + """ + headers_ = {'Accept': '*/*'} + netloc = parse_url(url).netloc + if netloc: + headers_['Host'] = netloc + if headers: + headers_.update(headers) + return headers_ + + def urlopen(self, method, url, redirect=True, **kw): + "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." + u = parse_url(url) + if u.scheme == "http": + # For proxied HTTPS requests, httplib sets the necessary headers + # on the CONNECT to the proxy. For HTTP, we'll definitely + # need to set 'Host' at the very least. + headers = kw.get('headers', self.headers) + kw['headers'] = self._set_proxy_headers(url, headers) + return super(ProxyManager, self).urlopen( + method, url, redirect=redirect, **kw + ) + + +def proxy_from_url(url, **kw): + return ProxyManager(proxy_url=url, **kw) diff --git a/requests3/core/_http/_async/response.py b/requests3/core/_http/_async/response.py new file mode 100644 index 00000000..78e6c264 --- /dev/null +++ b/requests3/core/_http/_async/response.py @@ -0,0 +1,461 @@ +from __future__ import absolute_import +from contextlib import contextmanager +import zlib +import io +import logging +from socket import timeout as SocketTimeout +from socket import error as SocketError + +import h11 + +from .._collections import HTTPHeaderDict +from ..exceptions import (ProtocolError, DecodeError, ReadTimeoutError) +from ..packages.six import string_types as basestring, binary_type +from ..util.ssl_ import BaseSSLError + +log = logging.getLogger(__name__) + + +class DeflateDecoder(object): + + def __init__(self): + self._first_try = True + self._data = binary_type() + self._obj = zlib.decompressobj() + + def __getattr__(self, name): + return getattr(self._obj, name) + + def decompress(self, data): + if not data: + return data + + if not self._first_try: + return self._obj.decompress(data) + + self._data += data + try: + decompressed = self._obj.decompress(data) + if decompressed: + self._first_try = False + self._data = None + return decompressed + + except zlib.error: + self._first_try = False + self._obj = zlib.decompressobj(-zlib.MAX_WBITS) + try: + return self.decompress(self._data) + + finally: + self._data = None + + +class GzipDecoder(object): + + def __init__(self): + self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) + + def __getattr__(self, name): + return getattr(self._obj, name) + + def decompress(self, data): + if not data: + return data + + return self._obj.decompress(data) + + +def _get_decoder(mode): + if mode == 'gzip': + return GzipDecoder() + + return DeflateDecoder() + + +class HTTPResponse(io.IOBase): + """ + HTTP Response container. + + Backwards-compatible to httplib's HTTPResponse but the response ``body`` is + loaded and decoded on-demand when the ``data`` property is accessed. This + class is also compatible with the Python standard library's :mod:`io` + module, and can hence be treated as a readable object in the context of that + framework. + + Extra parameters for behaviour not present in httplib.HTTPResponse: + + :param preload_content: + If True, the response's body will be preloaded during construction. + + :param decode_content: + If True, attempts to decode specific content-encoding's based on headers + (like 'gzip' and 'deflate') will be skipped and raw data will be used + instead. + + :param retries: + The retries contains the last :class:`~urllib3.util.retry.Retry` that + was used during the request. + """ + CONTENT_DECODERS = ['gzip', 'deflate'] + REDIRECT_STATUSES = [301, 302, 303, 307, 308] + + def __init__( + self, + body='', + headers=None, + status=0, + version=0, + reason=None, + strict=0, + preload_content=True, + decode_content=True, + original_response=None, + pool=None, + connection=None, + retries=None, + request_method=None, + ): + if isinstance(headers, HTTPHeaderDict): + self.headers = headers + else: + self.headers = HTTPHeaderDict(headers) + self.status = status + self.version = version + self.reason = reason + self.strict = strict + self.decode_content = decode_content + self.retries = retries + self._decoder = None + self._body = None + self._fp = None + self._original_response = original_response + self._fp_bytes_read = 0 + self._buffer = b'' + if body and isinstance(body, (basestring, binary_type)): + self._body = body + else: + self._fp = body + self._pool = pool + self._connection = connection + # If requested, preload the body. + if preload_content and not self._body: + self._body = self.read(decode_content=decode_content) + + def get_redirect_location(self): + """ + Should we redirect and where to? + + :returns: Truthy redirect location string if we got a redirect status + code and valid location. ``None`` if redirect status and no + location. ``False`` if not a redirect status code. + """ + if self.status in self.REDIRECT_STATUSES: + return self.headers.get('location') + + return False + + async def release_conn(self): + if not self._pool or not self._connection: + return + + await self._pool._put_conn(self._connection) + self._connection = None + + @property + def data(self): + # For backwords-compat with earlier urllib3 0.4 and earlier. + if self._body is not None: + return self._body + + if self._fp: + return self.read(cache_content=True) + + @property + def connection(self): + return self._connection + + def tell(self): + """ + Obtain the number of bytes pulled over the wire so far. May differ from + the amount of content returned by :meth:``HTTPResponse.read`` if bytes + are encoded on the wire (e.g, compressed). + """ + return self._fp_bytes_read + + def _init_decoder(self): + """ + Set-up the _decoder attribute if necessary. + """ + # Note: content-encoding value should be case-insensitive, per RFC 7230 + # Section 3.2 + content_encoding = self.headers.get('content-encoding', '').lower() + if self._decoder is None and content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) + + def _decode(self, data, decode_content, flush_decoder): + """ + Decode the data passed in and potentially flush the decoder. + """ + try: + if decode_content and self._decoder: + data = self._decoder.decompress(data) + except (IOError, zlib.error) as e: + content_encoding = self.headers.get('content-encoding', '').lower() + raise DecodeError( + "Received response with content-encoding: %s, but " + "failed to decode it." % content_encoding, + e, + ) + + if flush_decoder and decode_content: + data += self._flush_decoder() + return data + + def _flush_decoder(self): + """ + Flushes the decoder. Should only be called if the decoder is actually + being used. + """ + if self._decoder: + buf = self._decoder.decompress(b'') + return buf + self._decoder.flush() + + return b'' + + @contextmanager + def _error_catcher(self): + """ + Catch low-level python exceptions, instead re-raising urllib3 + variants, so that low-level exceptions are not leaked in the + high-level api. + + On exit, release the connection back to the pool. + """ + clean_exit = False + try: + try: + yield + + except SocketTimeout: + # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but + # there is yet no clean way to get at it from this context. + raise ReadTimeoutError(self._pool, None, 'Read timed out.') + + except BaseSSLError as e: + # FIXME: Is there a better way to differentiate between SSLErrors? + if 'read operation timed out' not in str(e): # Defensive: + # This shouldn't happen but just in case we're missing an edge + # case, let's avoid swallowing SSL errors. + raise + + raise ReadTimeoutError(self._pool, None, 'Read timed out.') + + except (h11.ProtocolError, SocketError) as e: + # This includes IncompleteRead. + raise ProtocolError('Connection broken: %r' % e, e) + + except GeneratorExit: + # We swallow GeneratorExit when it is emitted: this allows the + # use of the error checker inside stream() + pass + # If no exception is thrown, we should avoid cleaning up + # unnecessarily. + clean_exit = True + finally: + # If we didn't terminate cleanly, we need to throw away our + # connection. + if not clean_exit: + self.close() + # If we hold the original response but it's finished now, we should + # return the connection back to the pool. + # XXX + if False and self._original_response and self._original_response.complete: + self.release_conn() + + async def read(self, amt=None, decode_content=None, cache_content=False): + """ + Similar to :meth:`httplib.HTTPResponse.read`, but with two additional + parameters: ``decode_content`` and ``cache_content``. + + :param amt: + How much of the content to read. If specified, caching is skipped + because it doesn't make sense to cache partial content as the full + response. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + + :param cache_content: + If True, will save the returned data such that the same result is + returned despite of the state of the underlying file object. This + is useful if you want the ``.data`` property to continue working + after having ``.read()`` the file object. (Overridden if ``amt`` is + set.) + """ + # TODO: refactor this method to better handle buffered output. + # This method is a weird one. We treat this read() like a buffered + # read, meaning that it never reads "short" unless there is an EOF + # condition at work. However, we have a decompressor in play here, + # which means our read() returns decompressed data. + # + # This means the buffer can only meaningfully buffer decompressed data. + # This makes this method prone to over-reading, and forcing too much + # data into the buffer. That's unfortunate, but right now I'm not smart + # enough to come up with a way to solve that problem. + if self._fp is None and not self._buffer: + return b'' + + data = self._buffer + with self._error_catcher(): + if amt is None: + chunks = [] + async for chunk in self.stream(decode_content): + chunks.append(chunk) + data += b''.join(chunks) + self._buffer = b'' + # We only cache the body data for simple read calls. + self._body = data + else: + data_len = len(data) + chunks = [data] + streamer = self.stream(decode_content) + while data_len < amt: + try: + chunk = next(streamer) + except StopIteration: + break + + else: + chunks.append(chunk) + data_len += len(chunk) + data = b''.join(chunks) + self._buffer = data[amt:] + data = data[:amt] + return data + + async def stream(self, decode_content=None): + """ + A generator wrapper for the read() method. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + """ + # Short-circuit evaluation for exhausted responses. + if self._fp is None: + return + + self._init_decoder() + if decode_content is None: + decode_content = self.decode_content + with self._error_catcher(): + async for raw_chunk in self._fp: + self._fp_bytes_read += len(raw_chunk) + decoded_chunk = self._decode( + raw_chunk, decode_content, flush_decoder=False + ) + if decoded_chunk: + yield decoded_chunk + + # This branch is speculative: most decoders do not need to flush, + # and so this produces no output. However, it's here because + # anecdotally some platforms on which we do not test (like Jython) + # do require the flush. For this reason, we exclude this from code + # coverage. Happily, the code here is so simple that testing the + # branch we don't enter is basically entirely unnecessary (it's + # just a yield statement). + final_chunk = self._decode(b'', decode_content, flush_decoder=True) + if final_chunk: # Platform-specific: Jython + yield final_chunk + + self._fp = None + + @classmethod + def from_base(ResponseCls, r, **response_kw): + """ + Given an :class:`urllib3.base.Response` instance ``r``, return a + corresponding :class:`urllib3.response.HTTPResponse` object. + + Remaining parameters are passed to the HTTPResponse constructor, along + with ``original_response=r``. + """ + # TODO: Huge hack. + for kw in ('redirect', 'assert_same_host', 'enforce_content_length'): + if kw in response_kw: + response_kw.pop(kw) + + resp = ResponseCls( + body=r.body, + headers=r.headers, + status=r.status_code, + version=r.version, + original_response=r, + connection=r.body, + **response_kw + ) + return resp + + + # Backwards-compatibility methods for httplib.HTTPResponse + def getheaders(self): + return self.headers + + def getheader(self, name, default=None): + return self.headers.get(name, default) + + + # Backwards compatibility for http.cookiejar + def info(self): + return self.headers + + + # Overrides from io.IOBase + def close(self): + if not self.closed: + self._fp.close() + self._buffer = b'' + self._fp = None + if self._connection: + self._connection.close() + + @property + def closed(self): + # This method is required for `io` module compatibility. + if self._fp is None and not self._buffer: + return True + + elif hasattr(self._fp, 'complete'): + return self._fp.complete + + else: + return False + + def fileno(self): + # This method is required for `io` module compatibility. + if self._fp is None: + raise IOError("HTTPResponse has no file to get a fileno from") + + elif hasattr(self._fp, "fileno"): + return self._fp.fileno() + + else: + raise IOError( + "The file-like object this HTTPResponse is wrapped " + "around has no file descriptor" + ) + + def readable(self): + # This method is required for `io` module compatibility. + return True + + def readinto(self, b): + # This method is required for `io` module compatibility. + temp = self.read(len(b)) + if len(temp) == 0: + return 0 + + else: + b[:len(temp)] = temp + return len(temp) diff --git a/requests3/core/_http/_backends/__init__.py b/requests3/core/_http/_backends/__init__.py new file mode 100644 index 00000000..dbcc879d --- /dev/null +++ b/requests3/core/_http/_backends/__init__.py @@ -0,0 +1,9 @@ +from ..packages import six +from .sync_backend import SyncBackend + +__all__ = ['SyncBackend'] +if six.PY3: + from .trio_backend import TrioBackend + + from .twisted_backend import TwistedBackend + __all__ += ['TrioBackend', 'TwistedBackend'] diff --git a/requests3/core/_http/_backends/_common.py b/requests3/core/_http/_backends/_common.py new file mode 100644 index 00000000..62ef8397 --- /dev/null +++ b/requests3/core/_http/_backends/_common.py @@ -0,0 +1,29 @@ +from ..util import selectors + +__all__ = ["DEFAULT_SELECTOR", "is_readable", "LoopAbort"] +# We only ever select on 1 fd at a time, so there's no point in messing around +# with epoll/kqueue. But we do want to use PollSelector on platforms that have +# it (= everything except Windows), since it has no limit on the numerical +# value of the fds it accepts. On Windows, we use SelectSelector, but that's +# OK, because on Windows select also has no limit on the numerical value of +# the handles it accepts. +try: + selectors.PollSelector().select(timeout=0) +except (OSError, AttributeError): + DEFAULT_SELECTOR = selectors.SelectSelector +else: + DEFAULT_SELECTOR = selectors.PollSelector + + +def is_readable(sock): + s = DEFAULT_SELECTOR() + s.register(sock, selectors.EVENT_READ) + events = s.select(timeout=0) + return bool(events) + + +class LoopAbort(Exception): + """ + Tell backends that enough bytes have been consumed + """ + pass diff --git a/requests3/core/_http/_backends/sync_backend.py b/requests3/core/_http/_backends/sync_backend.py new file mode 100644 index 00000000..6332ff42 --- /dev/null +++ b/requests3/core/_http/_backends/sync_backend.py @@ -0,0 +1,136 @@ +import errno +import select +import socket +import ssl +from ..util.connection import create_connection +from ..util.ssl_ import ssl_wrap_socket +from ..util import selectors + +from ._common import DEFAULT_SELECTOR, is_readable, LoopAbort + +__all__ = ["SyncBackend"] +BUFSIZE = 65536 + + +class SyncBackend(object): + + def __init__(self, connect_timeout=None, read_timeout=None): + self._connect_timeout = connect_timeout + self._read_timeout = read_timeout + + def connect(self, host, port, source_address=None, socket_options=None): + conn = create_connection( + (host, port), + self._connect_timeout, + source_address=source_address, + socket_options=socket_options, + ) + return SyncSocket(conn, self._read_timeout) + + +class SyncSocket(object): + + def __init__(self, sock, read_timeout): + self._sock = sock + self._read_timeout = read_timeout + # We keep the socket in non-blocking mode, except during connect() and + # during the SSL handshake: + self._sock.setblocking(False) + + def start_tls(self, server_hostname, ssl_context): + self._sock.setblocking(True) + wrapped = ssl_wrap_socket( + self._sock, + server_hostname=server_hostname, + ssl_context=ssl_context, + ) + wrapped.setblocking(False) + return SyncSocket(wrapped, self._read_timeout) + + + # Only for SSL-wrapped sockets + def getpeercert(self, binary=False): + return self._sock.getpeercert(binary_form=binary) + + def _wait(self, readable, writable): + assert readable or writable + s = DEFAULT_SELECTOR() + flags = 0 + if readable: + flags |= selectors.EVENT_READ + if writable: + flags |= selectors.EVENT_WRITE + s.register(self._sock, flags) + events = s.select(timeout=self._read_timeout) + if not events: + raise socket.timeout("XX FIXME timeout happened") + + _, event = events[0] + return (event & selectors.EVENT_READ, event & selectors.EVENT_WRITE) + + def receive_some(self): + while True: + try: + return self._sock.recv(BUFSIZE) + + except ssl.SSLWantReadError: + self._wait(readable=True, writable=False) + except ssl.SSLWantWriteError: + self._wait(readable=False, writable=True) + except (OSError, socket.error) as exc: + if exc.errno in (errno.EWOULDBLOCK, errno.EAGAIN): + self._wait(readable=True, writable=False) + else: + raise + + def send_and_receive_for_a_while(self, produce_bytes, consume_bytes): + outgoing_finished = False + outgoing = b"" + try: + while True: + if not outgoing_finished and not outgoing: + # Can exit loop here with error + b = produce_bytes() + if b is None: + outgoing = None + outgoing_finished = True + else: + outgoing = memoryview(b) + want_read = False + want_write = False + try: + incoming = self._sock.recv(BUFSIZE) + except ssl.SSLWantReadError: + want_read = True + except ssl.SSLWantWriteError: + want_write = True + except (OSError, socket.error) as exc: + if exc.errno in (errno.EWOULDBLOCK, errno.EAGAIN): + want_read = True + else: + # Can exit loop here with LoopAbort + consume_bytes(incoming) + if not outgoing_finished: + try: + sent = self._sock.send(outgoing) + outgoing = outgoing[sent:] + except ssl.SSLWantReadError: + want_read = True + except ssl.SSLWantWriteError: + want_write = True + except (OSError, socket.error) as exc: + if exc.errno in (errno.EWOULDBLOCK, errno.EAGAIN): + want_write = True + if want_read or want_write: + self._wait(want_read, want_write) + except LoopAbort: + pass + + def forceful_close(self): + self._sock.close() + + def is_readable(self): + return is_readable(self._sock) + + def set_readable_watch_state(self, enabled): + pass diff --git a/requests3/core/_http/_backends/trio_backend.py b/requests3/core/_http/_backends/trio_backend.py new file mode 100644 index 00000000..c2af2138 --- /dev/null +++ b/requests3/core/_http/_backends/trio_backend.py @@ -0,0 +1,102 @@ +import trio + +from ._common import is_readable, LoopAbort + +BUFSIZE = 65536 + + +class TrioBackend: + + async def connect( + self, host, port, source_address=None, socket_options=None + ): + if source_address is not None: + # You can't really combine source_address= and happy eyeballs + # (can we get rid of source_address? or at least make it a source + # ip, no port?) + raise NotImplementedError( + "trio backend doesn't support setting source_address" + ) + + stream = await trio.open_tcp_stream(host, port) + for (level, optname, value) in socket_options: + stream.setsockopt(level, optname, value) + return TrioSocket(stream) + + def __len__(self): + return 1 + + def __gt__(self, other): + return len(self) > other + + + + +# XX it turns out that we don't need SSLStream to be robustified against +# cancellation, but we probably should do something to detect when the stream +# has been broken by cancellation (e.g. a timeout) and make is_readable return +# True so the connection won't be reused. +class TrioSocket: + + def __init__(self, stream): + self._stream = stream + + async def start_tls(self, server_hostname, ssl_context): + wrapped = trio.ssl.SSLStream( + self._stream, + ssl_context, + server_hostname=server_hostname, + https_compatible=True, + ) + return TrioSocket(wrapped) + + def getpeercert(self, binary=False): + return self._stream.getpeercert(binary=binary) + + async def receive_some(self): + return await self._stream.receive_some(BUFSIZE) + + async def send_and_receive_for_a_while(self, produce_bytes, consume_bytes): + + async def sender(): + while True: + outgoing = await produce_bytes() + if outgoing is None: + break + + await self._stream.send_all(outgoing) + + async def receiver(): + while True: + incoming = await self._stream.receive_some(BUFSIZE) + consume_bytes(incoming) + + try: + async with trio.open_nursery() as nursery: + nursery.start_soon(sender) + nursery.start_soon(receiver) + except LoopAbort: + pass + + + # Pull out the underlying trio socket, because it turns out HTTP is not so + # great at respecting abstraction boundaries. + def _socket(self): + stream = self._stream + # Strip off any layers of SSLStream + while hasattr(stream, "transport_stream"): + stream = stream.transport_stream + # Now we have a SocketStream + return stream.socket + + + # We want this to be synchronous, and don't care about graceful teardown + # of the SSL/TLS layer. + def forceful_close(self): + self._socket().close() + + def is_readable(self): + return is_readable(self._socket()) + + def set_readable_watch_state(self, enabled): + pass diff --git a/requests3/core/_http/_backends/twisted_backend.py b/requests3/core/_http/_backends/twisted_backend.py new file mode 100644 index 00000000..974b0dca --- /dev/null +++ b/requests3/core/_http/_backends/twisted_backend.py @@ -0,0 +1,272 @@ +import socket +import OpenSSL.crypto +from twisted.internet import protocol, ssl +from twisted.internet.interfaces import IHandshakeListener +from twisted.internet.endpoints import HostnameEndpoint, connectProtocol +from twisted.internet.defer import ( + Deferred, DeferredList, CancelledError, ensureDeferred +) +from zope.interface import implementer + +from ..contrib.pyopenssl import get_subj_alt_name +from ._common import LoopAbort + + + +# XX need to add timeout support, esp. on connect +class TwistedBackend: + + def __init__(self, reactor): + self._reactor = reactor + + async def connect( + self, host, port, source_address=None, socket_options=None + ): + # HostnameEndpoint only supports setting source host, not source port + if source_address is not None: + raise NotImplementedError( + "twisted backend doesn't support setting source_address" + ) + + # factory = protocol.Factory.forProtocol(TwistedSocketProtocol) + endpoint = HostnameEndpoint(self._reactor, host, port) + d = connectProtocol(endpoint, TwistedSocketProtocol()) + # XX d.addTimeout(...) + protocol = await d + if socket_options is not None: + for opt in socket_options: + if opt[:2] == (socket.IPPROTO_TCP, socket.TCP_NODELAY): + protocol.transport.setTcpNoDelay(opt[2]) + else: + raise NotImplementedError( + "unrecognized socket option for twisted backend" + ) + + return TwistedSocket(protocol) + + + + +# enums +class _DATA_RECEIVED: + pass + + +class _RESUME_PRODUCING: + pass + + +class _HANDSHAKE_COMPLETED: + pass + + +@implementer(IHandshakeListener) +class TwistedSocketProtocol(protocol.Protocol): + + def connectionMade(self): + self._receive_buffer = bytearray() + self.transport.pauseProducing() + self.transport.registerProducer(self, True) + self._producing = True + self._readable_watch_state_enabled = False + self._is_readable = False + self._events = {} + self._connection_lost = False + + def _signal(self, event): + if event in self._events: + # The first thing callback() will do is remove the deferred from + # self._events (see cleanup() in _wait_for() below). + self._events[event].callback(None) + + async def _wait_for(self, event): + assert event not in self._events + d = Deferred() + + # We might get callbacked, we might get cancelled; either way we want + # to clean up then pass through the result: + def cleanup(obj): + assert self._events[event] is d + del self._events[event] + return obj + + d.addBoth(cleanup) + self._events[event] = d + await d + + def dataReceived(self, data): + if self._readable_watch_state_enabled: + self._is_readable = True + self.transport.pauseProducing() + return + + self._receive_buffer += data + self._signal(_DATA_RECEIVED) + + def connectionLost(self, reason): + if self._readable_watch_state_enabled: + self._is_readable = True + self.transport.pauseProducing() + return + + self._connection_lost = True + self._signal(_DATA_RECEIVED) + + def pauseProducing(self): + self._producing = False + + def resumeProducing(self): + self._producing = True + self._signal(_RESUME_PRODUCING) + + def stopProducing(self): + pass + + def handshakeCompleted(self): + self._signal(_HANDSHAKE_COMPLETED) + + async def start_tls(self, server_hostname, ssl_context): + # XX ssl_context? + self.transport.startTLS(ssl.optionsForClientTLS(server_hostname)) + await self._wait_for(_HANDSHAKE_COMPLETED) + + async def receive_some(self): + assert not self._readable_watch_state_enabled + while not self._receive_buffer and not self._connection_lost: + self.transport.resumeProducing() + try: + await self._wait_for(_DATA_RECEIVED) + finally: + self.transport.pauseProducing() + got = self._receive_buffer + self._receive_buffer = bytearray() + return got + + async def send_all(self, data): + assert not self._readable_watch_state_enabled + while not self._producing: + await self._wait_for(_RESUME_PRODUCING) + self.transport.write(data) + + def is_readable(self): + assert self._readable_watch_state_enabled + return self._is_readable + + def set_readable_watch_state(self, enabled): + self._readable_watch_state_enabled = enabled + if self._readable_watch_state_enabled: + self.transport.resumeProducing() + else: + self.transport.pauseProducing() + + +class DoubleError(Exception): + + def __init__(self, exc1, exc2): + self.exc1 = exc1 + self.exc2 = exc2 + + def __str__(self): + return "{}, {}".format(self.exc1, self.exc2) + + +class TwistedSocket: + + def __init__(self, protocol): + self._protocol = protocol + + async def start_tls(self, server_hostname, ssl_context): + await self._protocol.start_tls(server_hostname, ssl_context) + + def getpeercert(self, binary=False): + # Cribbed from urllib3.contrib.pyopenssl.WrappedSocket.getpeercert + x509 = self._protocol.transport.getPeerCertificate() + if not x509: + return x509 + + if binary: + return OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_ASN1, x509 + ) + + return { + "subject": ((("commonName", x509.get_subject().CN),),), + "subjectAltName": get_subj_alt_name(x509), + } + + async def receive_some(self): + return await self._protocol.receive_some() + + async def send_and_receive_for_a_while(self, produce_bytes, consume_bytes): + + async def sender(): + while True: + outgoing = await produce_bytes() + if outgoing is None: + break + + await self._protocol.send_all(outgoing) + + async def receiver(): + while True: + incoming = await self._protocol.receive_some() + try: + consume_bytes(incoming) + except LoopAbort: + break + + # Run the two async functions concurrently + send_loop = ensureDeferred(sender()) + receive_loop = ensureDeferred(receiver()) + + # If the send_loop errors out, then cancel receive_loop and preserve + # the failure + @send_loop.addErrback + def send_loop_errback(failure): + receive_loop.cancel() + return failure + + + # If the receive_loop errors out *or* exits cleanly due to LoopAbort, + # then cancel the send_loop and preserve the result + @receive_loop.addBoth + def receive_loop_allback(result): + send_loop.cancel() + return result + + # Wait for both to finish, and then figure out if we need to raise an + # exception. + results = await DeferredList([send_loop, receive_loop]) + # First, find the failure objects - but since we've almost always + # cancelled one of the deferreds, which causes it to raise + # CancelledError, we can't treat these at face value. + failures = [] + for success, result in results: + if not success: + failures.append(result) + # First, loop over and remove at most 1 CancelledError, since that's + # the most that we ever generate. (If *we* were cancelled, then there + # will be 2 CancelledErrors, and that's fine; in that case we want to + # preserve 1 of them and then re-raise it.) + for i in range(len(failures)): + if isinstance(failures[i].value, CancelledError): + del failures[i] + break + + # Now whatever's left is what we need to re-raise + if len(failures) == 0: + return + + elif len(failures) == 1: + failures[0].raiseException() + else: + raise DoubleError(*failures) + + def forceful_close(self): + self._protocol.transport.abortConnection() + + def is_readable(self): + return self._protocol.is_readable() + + def set_readable_watch_state(self, enabled): + return self._protocol.set_readable_watch_state(enabled) diff --git a/requests3/core/_http/_collections.py b/requests3/core/_http/_collections.py new file mode 100644 index 00000000..8021ae20 --- /dev/null +++ b/requests3/core/_http/_collections.py @@ -0,0 +1,334 @@ +from __future__ import absolute_import + +try: + from collections.abc import Mapping, MutableMapping +except ImportError: + from collections import Mapping, MutableMapping +try: + from threading import RLock +except ImportError: # Platform-specific: No threads available + + class RLock: + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_value, traceback): + pass + + +try: # Python 2.7+ + from collections import OrderedDict +except ImportError: + from .packages.ordered_dict import OrderedDict +from .exceptions import InvalidHeader +from .packages.six import iterkeys, itervalues, PY3 + +__all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict'] +_Null = object() + + +class RecentlyUsedContainer(MutableMapping): + """ + Provides a thread-safe dict-like container which maintains up to + ``maxsize`` keys while throwing away the least-recently-used keys beyond + ``maxsize``. + + :param maxsize: + Maximum number of recent elements to retain. + + :param dispose_func: + Every time an item is evicted from the container, + ``dispose_func(value)`` is called. Callback which will get called + """ + ContainerCls = OrderedDict + + def __init__(self, maxsize=10, dispose_func=None): + self._maxsize = maxsize + self.dispose_func = dispose_func + self._container = self.ContainerCls() + self.lock = RLock() + + def __getitem__(self, key): + # Re-insert the item, moving it to the end of the eviction line. + with self.lock: + item = self._container.pop(key) + self._container[key] = item + return item + + def __setitem__(self, key, value): + evicted_value = _Null + with self.lock: + # Possibly evict the existing value of 'key' + evicted_value = self._container.get(key, _Null) + self._container[key] = value + # If we didn't evict an existing value, we might have to evict the + # least recently used item from the beginning of the container. + if len(self._container) > self._maxsize: + _key, evicted_value = self._container.popitem(last=False) + if self.dispose_func and evicted_value is not _Null: + self.dispose_func(evicted_value) + + def __delitem__(self, key): + with self.lock: + value = self._container.pop(key) + if self.dispose_func: + self.dispose_func(value) + + def __len__(self): + with self.lock: + return len(self._container) + + def __iter__(self): + raise NotImplementedError( + 'Iteration over this class is unlikely to be threadsafe.' + ) + + def clear(self): + with self.lock: + # Copy pointers to all values, then wipe the mapping + values = list(itervalues(self._container)) + self._container.clear() + if self.dispose_func: + for value in values: + self.dispose_func(value) + + def keys(self): + with self.lock: + return list(iterkeys(self._container)) + + +class HTTPHeaderDict(MutableMapping): + """ + :param headers: + An iterable of field-value pairs. Must not contain multiple field names + when compared case-insensitively. + + :param kwargs: + Additional field-value pairs to pass in to ``dict.update``. + + A ``dict`` like container for storing HTTP Headers. + + Field names are stored and compared case-insensitively in compliance with + RFC 7230. Iteration provides the first case-sensitive key seen for each + case-insensitive pair. + + Using ``__setitem__`` syntax overwrites fields that compare equal + case-insensitively in order to maintain ``dict``'s api. For fields that + compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add`` + in a loop. + + If multiple fields that are equal case-insensitively are passed to the + constructor or ``.update``, the behavior is undefined and some will be + lost. + + >>> headers = HTTPHeaderDict() + >>> headers.add('Set-Cookie', 'foo=bar') + >>> headers.add('set-cookie', 'baz=quxx') + >>> headers['content-length'] = '7' + >>> headers['SET-cookie'] + 'foo=bar, baz=quxx' + >>> headers['Content-Length'] + '7' + """ + + def __init__(self, headers=None, **kwargs): + super(HTTPHeaderDict, self).__init__() + self._container = OrderedDict() + if headers is not None: + if isinstance(headers, HTTPHeaderDict): + self._copy_from(headers) + else: + self.extend(headers) + if kwargs: + self.extend(kwargs) + + def __setitem__(self, key, val): + self._container[key.lower()] = [key, val] + return self._container[key.lower()] + + def __getitem__(self, key): + val = self._container[key.lower()] + return ', '.join(val[1:]) + + def __delitem__(self, key): + del self._container[key.lower()] + + def __contains__(self, key): + return key.lower() in self._container + + def __eq__(self, other): + if not isinstance(other, Mapping) and not hasattr(other, 'keys'): + return False + + if not isinstance(other, type(self)): + other = type(self)(other) + return ( + dict((k.lower(), v) for k, v in self.itermerged()) == + dict((k.lower(), v) for k, v in other.itermerged()) + ) + + def __ne__(self, other): + return not self.__eq__(other) + + if not PY3: # Python 2 + iterkeys = MutableMapping.iterkeys + itervalues = MutableMapping.itervalues + __marker = object() + + def __len__(self): + return len(self._container) + + def __iter__(self): + # Only provide the originally cased names + for vals in self._container.values(): + yield vals[0] + + def pop(self, key, default=__marker): + '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. + If key is not found, d is returned if given, otherwise KeyError is raised. + ''' + # Using the MutableMapping function directly fails due to the private marker. + # Using ordinary dict.pop would expose the internal structures. + # So let's reinvent the wheel. + try: + value = self[key] + except KeyError: + if default is self.__marker: + raise + + return default + + else: + del self[key] + return value + + def discard(self, key): + try: + del self[key] + except KeyError: + pass + + def add(self, key, val): + """Adds a (name, value) pair, doesn't overwrite the value if it already + exists. + + >>> headers = HTTPHeaderDict(foo='bar') + >>> headers.add('Foo', 'baz') + >>> headers['foo'] + 'bar, baz' + """ + key_lower = key.lower() + new_vals = [key, val] + # Keep the common case aka no item present as fast as possible + vals = self._container.setdefault(key_lower, new_vals) + if new_vals is not vals: + vals.append(val) + + def extend(self, *args, **kwargs): + """Generic import function for any type of header-like object. + Adapted version of MutableMapping.update in order to insert items + with self.add instead of self.__setitem__ + """ + if len(args) > 1: + raise TypeError( + "extend() takes at most 1 positional " + "arguments ({0} given)".format(len(args)) + ) + + other = args[0] if len(args) >= 1 else () + if isinstance(other, HTTPHeaderDict): + for key, val in other.iteritems(): + self.add(key, val) + elif isinstance(other, Mapping): + for key in other: + self.add(key, other[key]) + elif hasattr(other, "keys"): + for key in other.keys(): + self.add(key, other[key]) + else: + for key, value in other: + self.add(key, value) + for key, value in kwargs.items(): + self.add(key, value) + + def getlist(self, key, default=__marker): + """Returns a list of all the values for the named field. Returns an + empty list if the key doesn't exist.""" + try: + vals = self._container[key.lower()] + except KeyError: + if default is self.__marker: + return [] + + return default + + else: + return vals[1:] + + # Backwards compatibility for httplib + getheaders = getlist + getallmatchingheaders = getlist + iget = getlist + # Backwards compatibility for http.cookiejar + get_all = getlist + + def __repr__(self): + return "%s(%s)" % (type(self).__name__, dict(self.itermerged())) + + def _copy_from(self, other): + for key in other: + val = other.getlist(key) + if isinstance(val, list): + # Don't need to convert tuples + val = list(val) + self._container[key.lower()] = [key] + val + + def copy(self): + clone = type(self)() + clone._copy_from(self) + return clone + + def iteritems(self): + """Iterate over all header lines, including duplicate ones.""" + for key in self: + vals = self._container[key.lower()] + for val in vals[1:]: + yield vals[0], val + + def itermerged(self): + """Iterate over all headers, merging duplicate ones together.""" + for key in self: + val = self._container[key.lower()] + yield val[0], ', '.join(val[1:]) + + def items(self): + return list(self.iteritems()) + + @classmethod + def from_httplib(cls, message): # Python 2 + """Read headers from a Python 2 httplib message object.""" + # python2.7 does not expose a proper API for exporting multiheaders + # efficiently. This function re-reads raw lines from the message + # object and extracts the multiheaders properly. + obs_fold_continued_leaders = (' ', '\t') + headers = [] + for line in message.headers: + if line.startswith(obs_fold_continued_leaders): + if not headers: + # We received a header line that starts with OWS as described + # in RFC-7230 S3.2.4. This indicates a multiline header, but + # there exists no previous header to which we can attach it. + raise InvalidHeader( + 'Header continuation with no previous header: %s' % + line + ) + + else: + key, value = headers[-1] + headers[-1] = (key, value + ' ' + line.strip()) + continue + + key, value = line.split(':', 1) + headers.append((key, value.strip())) + return cls(headers) diff --git a/requests3/core/_http/_sync/__init__.py b/requests3/core/_http/_sync/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/requests3/core/_http/_sync/connection.py b/requests3/core/_http/_sync/connection.py new file mode 100644 index 00000000..fbfa5ab9 --- /dev/null +++ b/requests3/core/_http/_sync/connection.py @@ -0,0 +1,526 @@ +# -*- coding: utf-8 -*- +""" +This module implements the connection management logic. + +Unlike in http.client, the connection here is an object that is responsible +for a very small number of tasks: + + 1. Serializing/deserializing data to/from the network. + 2. Being able to do basic parsing of HTTP and maintaining the framing. + 3. Understanding connection state. + +This object knows very little about the semantics of HTTP in terms of how to +construct HTTP requests and responses. It mostly manages the socket itself. +""" +from __future__ import absolute_import + +import collections +import datetime +import socket +import warnings + +import h11 + +from ..base import Request, Response +from ..exceptions import ( + ConnectTimeoutError, + NewConnectionError, + SubjectAltNameWarning, + SystemTimeWarning, + BadVersionError, + FailedTunnelError, + InvalidBodyError, + ProtocolError, +) +from ..packages import six +from ..util import ssl_ as ssl_util +from .._backends import SyncBackend +from .._backends._common import LoopAbort + +try: + import ssl +except ImportError: + ssl = None +# When updating RECENT_DATE, move it to +# within two years of the current date, and no +# earlier than 6 months ago. +RECENT_DATE = datetime.date(2016, 1, 1) +_SUPPORTED_VERSIONS = frozenset([b'1.0', b'1.1']) +# A sentinel object returned when some syscalls return EAGAIN. +_EAGAIN = object() + + +def _headers_to_native_string(headers): + """ + A temporary shim to convert received headers to native strings, to match + the behaviour of httplib. We will reconsider this later in the process. + """ + # TODO: revisit. + # This works because fundamentally we know that all headers coming from + # h11 are bytes, so if they aren't of type `str` then we must be on Python + # 3 and need to decode the headers using Latin1. + for n, v in headers: + if not isinstance(n, str): + n = n.decode('latin1') + if not isinstance(v, str): + v = v.decode('latin1') + yield (n, v) + + +def _stringify_headers(headers): + """ + A generator that transforms headers so they're suitable for sending by h11. + """ + # TODO: revisit + for name, value in headers: + if isinstance(name, six.text_type): + name = name.encode('ascii') + if isinstance(value, six.text_type): + value = value.encode('latin-1') + elif isinstance(value, int): + value = str(value).encode('ascii') + yield (name, value) + + +def _read_readable(readable): + # TODO: reconsider this block size + blocksize = 8192 + while True: + datablock = readable.read(blocksize) + if not datablock: + break + + yield datablock + + + + +# XX this should return an async iterator +def _make_body_iterable(body): + """ + This function turns all possible body types that urllib3 supports into an + iterable of bytes. The goal is to expose a uniform structure to request + bodies so that they all appear to be identical to the low-level code. + + The basic logic here is: + - byte strings are turned into single-element lists + - readables are wrapped in an iterable that repeatedly calls read until + nothing is returned anymore + - other iterables are used directly + - anything else is not acceptable + + In particular, note that we do not support *text* data of any kind. This + is deliberate: users must make choices about the encoding of the data they + use. + """ + if body is None: + return [] + + elif isinstance(body, six.binary_type): + return [body] + + elif hasattr(body, "read"): + return _read_readable(body) + + elif isinstance(body, collections.Iterable) and not isinstance( + body, six.text_type + ): + return body + + else: + raise InvalidBodyError("Unacceptable body type: %s" % type(body)) + + + + +# XX this should return an async iterator +def _request_bytes_iterable(request, state_machine): + """ + An iterable that serialises a set of bytes for the body. + """ + h11_request = h11.Request( + method=request.method, + target=request.target, + headers=_stringify_headers(request.headers.items()), + ) + yield state_machine.send(h11_request) + + for chunk in _make_body_iterable(request.body): + yield state_machine.send(h11.Data(data=chunk)) + + yield state_machine.send(h11.EndOfMessage()) + + +def _response_from_h11(h11_response, body_object): + """ + Given a h11 Response object, build a urllib3 response object and return it. + """ + if h11_response.http_version not in _SUPPORTED_VERSIONS: + raise BadVersionError(h11_response.http_version) + + version = b'HTTP/' + h11_response.http_version + our_response = Response( + status_code=h11_response.status_code, + headers=_headers_to_native_string(h11_response.headers), + body=body_object, + version=version, + ) + return our_response + + +def _build_tunnel_request(host, port, headers): + """ + Builds a urllib3 Request object that is set up correctly to request a proxy + to establish a TCP tunnel to the remote host. + """ + target = "%s:%d" % (host, port) + if not isinstance(target, bytes): + target = target.encode('latin1') + tunnel_request = Request(method=b"CONNECT", target=target, headers=headers) + tunnel_request.add_host(host=host, port=port, scheme='http') + return tunnel_request + + +def _start_http_request(request, state_machine, conn): + """ + Send the request using the given state machine and connection, wait + for the response headers, and return them. + + If we get response headers early, then we stop sending and return + immediately, poisoning the state machine along the way so that we know + it can't be re-used. + + This is a standalone function because we use it both to set up both + CONNECT requests and real requests. + """ + # Before we begin, confirm that the state machine is ok. + if ( + state_machine.our_state is not h11.IDLE or + state_machine.their_state is not h11.IDLE + ): + raise ProtocolError("Invalid internal state transition") + + request_bytes_iterable = _request_bytes_iterable(request, state_machine) + # Hack around Python 2 lack of nonlocal + context = {'send_aborted': True, 'h11_response': None} + + def next_bytes_to_send(): + try: + return next(request_bytes_iterable) + + except StopIteration: + # We successfully sent the whole body! + context['send_aborted'] = False + return None + + def consume_bytes(data): + state_machine.receive_data(data) + while True: + event = state_machine.next_event() + if event is h11.NEED_DATA: + break + + elif isinstance(event, h11.InformationalResponse): + # Ignore 1xx responses + continue + + elif isinstance(event, h11.Response): + # We have our response! Save it and get out of here. + context['h11_response'] = event + raise LoopAbort + + else: + # Can't happen + raise RuntimeError("Unexpected h11 event {}".format(event)) + + conn.send_and_receive_for_a_while(next_bytes_to_send, consume_bytes) + assert context['h11_response'] is not None + if context['send_aborted']: + # Our state machine thinks we sent a bunch of data... but maybe we + # didn't! Maybe our send got cancelled while we were only half-way + # through sending the last chunk, and then h11 thinks we sent a + # complete request and we actually didn't. Then h11 might think we can + # re-use this connection, even though we can't. So record this in + # h11's state machine. + # XX need to implement this in h11 + # state_machine.poison() + # XX kluge for now + state_machine._cstate.process_error(state_machine.our_role) + return context['h11_response'] + + +def _read_until_event(state_machine, conn): + """ + A loop that keeps issuing reads and feeding the data into h11 and + checking whether h11 has an event for us. The moment there is an event + other than h11.NEED_DATA, this function returns that event. + """ + while True: + event = state_machine.next_event() + if event is not h11.NEED_DATA: + return event + + state_machine.receive_data(conn.receive_some()) + + +_DEFAULT_SOCKET_OPTIONS = object() + + +class HTTP1Connection(object): + """ + A wrapper around a single HTTP/1.1 connection. + + This wrapper manages connection state, ensuring that connections are + appropriately managed throughout the lifetime of a HTTP transaction. In + particular, this object understands the conditions in which connections + should be torn down, and also manages sending data and handling early + responses. + + This object can be iterated over to return the response body. When iterated + over it will return all of the data that is currently buffered, and if no + data is buffered it will issue one read syscall and return all of that + data. Buffering of response data must happen at a higher layer. + """ + # : Disable Nagle's algorithm by default. + #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]`` + default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)] + + def __init__( + self, + host, + port, + backend=None, + socket_options=_DEFAULT_SOCKET_OPTIONS, + source_address=None, + tunnel_host=None, + tunnel_port=None, + tunnel_headers=None, + ): + self.is_verified = False + self._backend = backend or SyncBackend() + self._host = host + self._port = port + self._socket_options = ( + socket_options if socket_options is not _DEFAULT_SOCKET_OPTIONS else self.default_socket_options + ) + self._source_address = source_address + self._tunnel_host = tunnel_host + self._tunnel_port = tunnel_port + self._tunnel_headers = tunnel_headers + self._sock = None + self._state_machine = h11.Connection(our_role=h11.CLIENT) + + def _wrap_socket( + self, conn, ssl_context, fingerprint, assert_hostname + ): + """ + Handles extra logic to wrap the socket in TLS magic. + """ + is_time_off = datetime.date.today() < RECENT_DATE + if is_time_off: + warnings.warn( + ( + 'System time is way off (before {0}). This will probably ' + 'lead to SSL verification errors' + ).format( + RECENT_DATE + ), + SystemTimeWarning, + ) + # XX need to know whether this is the proxy or the final host that + # we just did a handshake with! + check_host = assert_hostname or self._tunnel_host or self._host + # Stripping trailing dots from the hostname is important because + # they indicate that this host is an absolute name (for DNS + # lookup), but are irrelevant to SSL hostname matching and in fact + # will break it. + check_host = check_host.rstrip(".") + conn = conn.start_tls(check_host, ssl_context) + if fingerprint: + ssl_util.assert_fingerprint( + conn.getpeercert(binary_form=True), fingerprint + ) + elif ( + ssl_context.verify_mode != ssl.CERT_NONE and + assert_hostname is not False + ): + cert = conn.getpeercert() + if not cert.get('subjectAltName', ()): + warnings.warn( + ( + 'Certificate for {0} has no `subjectAltName`, falling ' + 'back to check for a `commonName` for now. This ' + 'feature is being removed by major browsers and ' + 'deprecated by RFC 2818. (See ' + 'https://github.com/shazow/urllib3/issues/497 for ' + 'details.)'.format(self._host) + ), + SubjectAltNameWarning, + ) + ssl_util.match_hostname(cert, check_host) + self.is_verified = ( + ssl_context.verify_mode == ssl.CERT_REQUIRED and + (assert_hostname is not False or fingerprint) + ) + return conn + + def send_request(self, request, read_timeout): + """ + Given a Request object, performs the logic required to get a response. + """ + h11_response = _start_http_request( + request, self._state_machine, self._sock + ) + return _response_from_h11(h11_response, self) + + def _tunnel(self, conn): + """ + This method establishes a CONNECT tunnel shortly after connection. + """ + # Basic sanity check that _tunnel is only called at appropriate times. + assert self._state_machine.our_state is h11.IDLE + tunnel_request = _build_tunnel_request( + self._tunnel_host, self._tunnel_port, self._tunnel_headers + ) + tunnel_state_machine = h11.Connection(our_role=h11.CLIENT) + h11_response = _start_http_request( + tunnel_request, tunnel_state_machine, conn + ) + # XX this is wrong -- 'self' here will try to iterate using + # self._state_machine, not tunnel_state_machine. Also, we need to + # think about how this failure case interacts with the pool's + # connection lifecycle management. + tunnel_response = _response_from_h11(h11_response, self) + if h11_response.status_code != 200: + conn.forceful_close() + raise FailedTunnelError( + "Unable to establish CONNECT tunnel", tunnel_response + ) + + def connect( + self, + ssl_context=None, + fingerprint=None, + assert_hostname=None, + connect_timeout=None, + ): + """ + Connect this socket to the server, applying the source address, any + relevant socket options, and the relevant connection timeout. + """ + if self._sock is not None: + # We're already connected, move on. + self._sock.set_readable_watch_state(False) + return + + extra_kw = {} + if self._source_address: + extra_kw['source_address'] = self._source_address + if self._socket_options: + extra_kw['socket_options'] = self._socket_options + # XX pass connect_timeout to backend + # This was factored out into a separate function to allow overriding + # by subclasses, but in the backend approach the way to to this is to + # provide a custom backend. (Composition >> inheritance.) + try: + conn = self._backend.connect( + self._host, self._port, **extra_kw + ) + # XX these two error handling blocks needs to be re-done in a + # backend-agnostic way + except socket.timeout: + raise ConnectTimeoutError( + self, + "Connection to %s timed out. (connect timeout=%s)" % + (self._host, connect_timeout), + ) + + except socket.error as e: + raise NewConnectionError( + self, "Failed to establish a new connection: %s" % e + ) + + if ssl_context is not None: + if self._tunnel_host is not None: + self._tunnel(conn) + conn = self._wrap_socket( + conn, ssl_context, fingerprint, assert_hostname + ) + # XX We should pick one of these names and use it consistently... + self._sock = conn + + def close(self): + """ + Close this connection. + """ + if self._sock is not None: + # Make sure self._sock is None even if closing raises an exception + sock, self._sock = self._sock, None + sock.forceful_close() + + def is_dropped(self): + """ + Returns True if the connection is closed: returns False otherwise. This + includes closures that do not mark the FD as closed, such as when the + remote peer has sent EOF but we haven't read it yet. + + Pre-condition: _reset must have been called. + """ + if self._sock is None: + return True + + # We check for droppedness by checking the socket for readability. If + # it's not readable, it's not dropped. If it is readable, then we + # assume that the thing we'd read from the socket is EOF. It might not + # be, but if it's not then the server has busted its HTTP/1.1 framing + # and so we want to drop the connection anyway. + return self._sock.is_readable() + + def _reset(self): + """ + Called once we hit EndOfMessage, and checks whether we can re-use this + state machine and connection or not, and if not, closes the socket and + state machine. + """ + try: + self._state_machine.start_next_cycle() + except h11.LocalProtocolError: + # Not re-usable + self.close() + else: + # This connection can be returned to the connection pool, and + # eventually we'll take it out again and want to know if it's been + # dropped. + self._sock.set_readable_watch_state(True) + + @property + def complete(self): + """ + XX what is this supposed to do? check if the response has been fully + iterated over? check for that + the connection being reusable? + """ + our_state = self._state_machine.our_state + their_state = self._state_machine.their_state + return (our_state is h11.IDLE and their_state is h11.IDLE) + + def __iter__(self): + return self + + def next(self): # Needed for Python 2 as __anext__ becomes __next__ + return self.__next__() + + def __next__(self): + """ + Iterate over the body bytes of the response until end of message. + """ + event = _read_until_event(self._state_machine, self._sock) + if isinstance(event, h11.Data): + return bytes(event.data) + + elif isinstance(event, h11.EndOfMessage): + self._reset() + raise StopIteration + + else: + # can't happen + raise RuntimeError("Unexpected h11 event {}".format(event)) diff --git a/requests3/core/_http/_sync/connectionpool.py b/requests3/core/_http/_sync/connectionpool.py new file mode 100644 index 00000000..e0bf5290 --- /dev/null +++ b/requests3/core/_http/_sync/connectionpool.py @@ -0,0 +1,891 @@ +from __future__ import absolute_import +import errno +import logging +import sys +import warnings + +from socket import error as SocketError, timeout as SocketTimeout +import socket + +import h11 + + +from ..base import Request, DEFAULT_PORTS +from ..exceptions import ( + ClosedPoolError, + ProtocolError, + EmptyPoolError, + LocationValueError, + MaxRetryError, + ProxyError, + ReadTimeoutError, + SSLError, + TimeoutError, + InsecureRequestWarning, + NewConnectionError, +) +from ..packages.ssl_match_hostname import CertificateError +from ..packages import six +from ..packages.six.moves import queue +from ..request import RequestMethods +from .response import HTTPResponse +from .connection import HTTP1Connection + +from ..util.connection import is_connection_dropped +from ..util.request import set_file_position +from ..util.retry import Retry +from ..util.ssl_ import ( + create_urllib3_context, + merge_context_settings, + resolve_ssl_version, + resolve_cert_reqs, + BaseSSLError, +) +from ..util.timeout import Timeout +from ..util.url import get_host, Url + +try: + import ssl +except ImportError: + ssl = None +if six.PY2: + # Queue is imported for side effects on MS Windows + import Queue as _unused_module_Queue # noqa: F401 +xrange = six.moves.xrange +log = logging.getLogger(__name__) +_Default = object() + + +def _add_transport_headers(headers): + """ + Adds the transport framing headers, if needed. Naturally, this method + cannot add a content-length header, so if there is no content-length header + then it will add Transfer-Encoding: chunked instead. Should only be called + if there is a body to upload. + + This should be a bit smarter: in particular, it should allow for bad or + unexpected versions of these headers, particularly transfer-encoding. + """ + transfer_headers = ('content-length', 'transfer-encoding') + for header_name in headers: + if header_name.lower() in transfer_headers: + return + + headers['transfer-encoding'] = 'chunked' + + +def _build_context( + context, keyfile, certfile, cert_reqs, ca_certs, ca_cert_dir, ssl_version +): + """ + Creates a urllib3 context suitable for a given request based on a + collection of possible properties of that context. + """ + if context is None: + context = create_urllib3_context( + ssl_version=resolve_ssl_version(ssl_version), + cert_reqs=resolve_cert_reqs(cert_reqs), + ) + context = merge_context_settings( + context, + keyfile=keyfile, + certfile=certfile, + cert_reqs=cert_reqs, + ca_certs=ca_certs, + ca_cert_dir=ca_cert_dir, + ) + return context + + + + +# Pool objects +class ConnectionPool(object): + """ + Base class for all connection pools, such as + :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`. + """ + scheme = None + QueueCls = queue.LifoQueue + + def __init__(self, host, port=None): + if not host: + raise LocationValueError("No host specified.") + + self.host = _ipv6_host(host).lower() + self.port = port + + def __str__(self): + return '%s(host=%r, port=%r)' % ( + type(self).__name__, self.host, self.port + ) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + # Return False to re-raise any potential exceptions + return False + + def close(self): + """ + Close all pooled connections and disable the pool. + """ + pass + + +# This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 +_blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK]) + + +class HTTPConnectionPool(ConnectionPool, RequestMethods): + """ + Thread-safe connection pool for one host. + + :param host: + Host used for this HTTP Connection (e.g. "localhost"), passed into + :class:`httplib.HTTPConnection`. + + :param port: + Port used for this HTTP Connection (None is equivalent to 80), passed + into :class:`httplib.HTTPConnection`. + + :param strict: + Causes BadStatusLine to be raised if the status line can't be parsed + as a valid HTTP/1.0 or 1.1 status line, passed into + :class:`httplib.HTTPConnection`. + + .. note:: + Only works in Python 2. This parameter is ignored in Python 3. + + :param timeout: + Socket timeout in seconds for each individual connection. This can + be a float or integer, which sets the timeout for the HTTP request, + or an instance of :class:`urllib3.util.Timeout` which gives you more + fine-grained control over request timeouts. After the constructor has + been parsed, this is always a `urllib3.util.Timeout` object. + + :param maxsize: + Number of connections to save that can be reused. More than 1 is useful + in multithreaded situations. If ``block`` is set to False, more + connections will be created but they will not be saved once they've + been used. + + :param block: + If set to True, no more than ``maxsize`` connections will be used at + a time. When no free connections are available, the call will block + until a connection has been released. This is a useful side effect for + particular multithreaded situations where one does not want to use more + than maxsize connections per host to prevent flooding. + + :param headers: + Headers to include with all requests, unless other headers are given + explicitly. + + :param retries: + Retry configuration to use by default with requests in this pool. + + :param _proxy: + Parsed proxy URL, should not be used directly, instead, see + :class:`urllib3.connectionpool.ProxyManager`" + + :param _proxy_headers: + A dictionary with proxy headers, should not be used directly, + instead, see :class:`urllib3.connectionpool.ProxyManager`" + + :param \\**conn_kw: + Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`, + :class:`urllib3.connection.HTTPSConnection` instances. + """ + scheme = 'http' + ConnectionCls = HTTP1Connection + ResponseCls = HTTPResponse + + def __init__( + self, + host, + port=None, + timeout=Timeout.DEFAULT_TIMEOUT, + maxsize=1, + block=False, + headers=None, + retries=None, + _proxy=None, + _proxy_headers=None, + **conn_kw + ): + ConnectionPool.__init__(self, host, port) + RequestMethods.__init__(self, headers) + if not isinstance(timeout, Timeout): + timeout = Timeout.from_float(timeout) + if retries is None: + retries = Retry.DEFAULT + self.timeout = timeout + self.retries = retries + self.pool = self.QueueCls(maxsize) + self.block = block + self.proxy = _proxy + self.proxy_headers = _proxy_headers or {} + # Fill the queue up so that doing get() on it will block properly + for _ in xrange(maxsize): + self.pool.put(None) + # These are mostly for testing and debugging purposes. + self.num_connections = 0 + self.num_requests = 0 + self.conn_kw = conn_kw + if self.proxy: + # Enable Nagle's algorithm for proxies, to avoid packet fragmentation. + # We cannot know if the user has added default socket options, so we cannot replace the + # list. + self.conn_kw.setdefault('socket_options', []) + + def _new_conn(self): + """ + Return a fresh connection. + """ + self.num_connections += 1 + + # TODO: Huge hack. + for kw in ('strict',): + if kw in self.conn_kw: + self.conn_kw.pop(kw) + + log.debug( + "Starting new HTTP connection (%d): %s:%s", + self.num_connections, + self.host, + self.port or "80", + ) + conn = self.ConnectionCls( + host=self.host, port=self.port, ** self.conn_kw + ) + return conn + + def _get_conn(self, timeout=None): + """ + Get a connection. Will return a pooled connection if one is available. + + If no connections are available and :prop:`.block` is ``False``, then a + fresh connection is returned. + + :param timeout: + Seconds to wait before giving up and raising + :class:`urllib3.exceptions.EmptyPoolError` if the pool is empty and + :prop:`.block` is ``True``. + """ + conn = None + try: + conn = self.pool.get(block=self.block, timeout=timeout) + except AttributeError: # self.pool is None + raise ClosedPoolError(self, "Pool is closed.") + + except queue.Empty: + if self.block: + raise EmptyPoolError( + self, + "Pool reached maximum size and no more " + "connections are allowed.", + ) + + pass # Oh well, we'll create a new connection then + # If this is a persistent connection, check if it got disconnected + if conn and is_connection_dropped(conn): + log.debug("Resetting dropped connection: %s", self.host) + conn.close() + return conn or self._new_conn() + + def _put_conn(self, conn): + """ + Put a connection back into the pool. + + :param conn: + Connection object for the current host and port as returned by + :meth:`._new_conn` or :meth:`._get_conn`. + + If the pool is already full, the connection is closed and discarded + because we exceeded maxsize. If connections are discarded frequently, + then maxsize should be increased. + + If the pool is closed, then the connection will be closed and discarded. + """ + try: + self.pool.put(conn, block=False) + return # Everything is dandy, done. + + except AttributeError: + # self.pool is None. + pass + except queue.Full: + # This should never happen if self.block == True + log.warning( + "Connection pool is full, discarding connection: %s", self.host + ) + # Connection never got put back into the pool, close it. + if conn: + conn.close() + + def _start_conn(self, conn, connect_timeout): + """ + Called right before a request is made, after the socket is created. + """ + conn.connect(connect_timeout=connect_timeout) + + def _get_timeout(self, timeout): + """ Helper that always returns a :class:`urllib3.util.Timeout` """ + if timeout is _Default: + return self.timeout.clone() + + if isinstance(timeout, Timeout): + return timeout.clone() + + else: + # User passed us an int/float. This is for backwards compatibility, + # can be removed later + return Timeout.from_float(timeout) + + def _raise_timeout(self, err, url, timeout_value): + """Is the error actually a timeout? Will raise a ReadTimeout or pass""" + if isinstance(err, SocketTimeout): + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % timeout_value + ) + + # See the above comment about EAGAIN in Python 3. In Python 2 we have + # to specifically catch it and throw the timeout error + if hasattr(err, 'errno') and err.errno in _blocking_errnos: + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % timeout_value + ) + + # Catch possible read timeouts thrown as SSL errors. If not the + # case, rethrow the original. We need to do this because of: + # http://bugs.python.org/issue10272 + # TODO: Can we remove this? + if 'timed out' in str(err) or 'did not complete (read)' in str( + err + ): # Python 2.6 + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % timeout_value + ) + + def _make_request( + self, conn, method, url, timeout=_Default, body=None, headers=None + ): + """ + Perform a request on a given urllib connection object taken from our + pool. + + :param conn: + a connection from one of our connection pools + + :param timeout: + Socket timeout in seconds for the request. This can be a + float or integer, which will set the same timeout value for + the socket connect and the socket read, or an instance of + :class:`urllib3.util.Timeout`, which gives you more fine-grained + control over your timeouts. + """ + self.num_requests += 1 + timeout_obj = self._get_timeout(timeout) + timeout_obj.start_connect() + # Trigger any extra validation we need to do. + try: + self._start_conn(conn, timeout_obj.connect_timeout) + except (SocketTimeout, BaseSSLError) as e: + # Py2 raises this as a BaseSSLError, Py3 raises it as socket timeout. + self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) + raise + + # TODO: We need to encapsulate our proxy logic in here somewhere. + request = Request( + method=method, target=url, headers=headers, body=body + ) + host = self.host + port = self.port + scheme = self.scheme + request.add_host(host, port, scheme) + # Reset the timeout for the recv() on the socket + read_timeout = timeout_obj.read_timeout + # In Python 3 socket.py will catch EAGAIN and return None when you + # try and read into the file pointer created by http.client, which + # instead raises a BadStatusLine exception. Instead of catching + # the exception and assuming all BadStatusLine exceptions are read + # timeouts, check for a zero timeout before making the request. + if read_timeout == 0: + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % read_timeout + ) + + if read_timeout is Timeout.DEFAULT_TIMEOUT: + read_timeout = socket.getdefaulttimeout() + # Receive the response from the server + try: + response = conn.send_request( + request, read_timeout=read_timeout + ) + except (SocketTimeout, BaseSSLError, SocketError) as e: + self._raise_timeout(err=e, url=url, timeout_value=read_timeout) + raise + + # AppEngine doesn't have a version attr. + http_version = getattr(conn, '_http_vsn_str', 'HTTP/?') + log.debug( + "%s://%s:%s \"%s %s %s\" %s", + self.scheme, + self.host, + self.port, + method, + url, + http_version, + response.status_code, + ) + return response + + def _absolute_url(self, path): + return Url( + scheme=self.scheme, host=self.host, port=self.port, path=path + ).url + + def close(self): + """ + Close all pooled connections and disable the pool. + """ + if self.pool is None: + return + + # Disable access to the pool + old_pool, self.pool = self.pool, None + try: + while True: + conn = old_pool.get(block=False) + if conn: + conn.close() + except queue.Empty: + pass # Done. + + def is_same_host(self, url): + """ + Check if the given ``url`` is a member of the same host as this + connection pool. + """ + if url.startswith('/'): + return True + + # TODO: Add optional support for socket.gethostbyname checking. + scheme, host, port = get_host(url) + host = _ipv6_host(host).lower() + # Use explicit default port for comparison when none is given + if self.port and not port: + port = DEFAULT_PORTS.get(scheme) + elif not self.port and port == DEFAULT_PORTS.get(scheme): + port = None + return (scheme, host, port) == (self.scheme, self.host, self.port) + + def urlopen( + self, + method, + url, + body=None, + headers=None, + retries=None, + timeout=_Default, + pool_timeout=None, + body_pos=None, + **response_kw + ): + """ + Get a connection from the pool and perform an HTTP request. This is the + lowest level call for making a request, so you'll need to specify all + the raw details. + + .. note:: + + More commonly, it's appropriate to use a convenience method provided + by :class:`.RequestMethods`, such as :meth:`request`. + + :param method: + HTTP request method (such as GET, POST, PUT, etc.) + + :param body: + Data to send in the request body (useful for creating + POST requests, see HTTPConnectionPool.post_url for + more convenience). + + :param headers: + Dictionary of custom headers to send, such as User-Agent, + If-None-Match, etc. If None, pool headers are used. If provided, + these headers completely replace any pool-specific headers. + + :param retries: + Configure the number of retries to allow before raising a + :class:`~urllib3.exceptions.MaxRetryError` exception. + + Pass ``None`` to retry until you receive a response. Pass a + :class:`~urllib3.util.retry.Retry` object for fine-grained control + over different types of retries. + Pass an integer number to retry connection errors that many times, + but no other types of errors. Pass zero to never retry. + + If ``False``, then retries are disabled and any exception is raised + immediately. Also, instead of raising a MaxRetryError on redirects, + the redirect response will be returned. + + :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. + + :param timeout: + If specified, overrides the default timeout for this one + request. It may be a float (in seconds) or an instance of + :class:`urllib3.util.Timeout`. + + :param pool_timeout: + If set and the pool is set to block=True, then this method will + block for ``pool_timeout`` seconds and raise EmptyPoolError if no + connection is available within the time period. + + :param int body_pos: + Position to seek to in file-like body in the event of a retry or + redirect. Typically this won't need to be set because urllib3 will + auto-populate the value when needed. + + :param \\**response_kw: + Additional parameters are passed to + :meth:`urllib3.response.HTTPResponse.from_httplib` + """ + if headers is None: + headers = self.headers + if not isinstance(retries, Retry): + retries = Retry.from_int( + retries, default=self.retries, redirect=False + ) + conn = None + # Track whether `conn` needs to be released before + # returning/raising/recursing. + release_this_conn = False + # Merge the proxy headers. Only do this in HTTP. We have to copy the + # headers dict so we can safely change it without those changes being + # reflected in anyone else's copy. + if self.scheme == 'http': + headers = headers.copy() + headers.update(self.proxy_headers) + # Must keep the exception bound to a separate variable or else Python 3 + # complains about UnboundLocalError. + err = None + # Keep track of whether we cleanly exited the except block. This + # ensures we do proper cleanup in finally. + clean_exit = False + # Rewind body position, if needed. Record current position + # for future rewinds in the event of a redirect/retry. + body_pos = set_file_position(body, body_pos) + if body is not None: + _add_transport_headers(headers) + try: + # Request a connection from the queue. + timeout_obj = self._get_timeout(timeout) + conn = self._get_conn(timeout=pool_timeout) + conn.timeout = timeout_obj.connect_timeout + # Make the request on the base connection object. + base_response = self._make_request( + conn, + method, + url, + timeout=timeout_obj, + body=body, + headers=headers, + ) + # Pass method to Response for length checking + response_kw['request_method'] = method + # Import httplib's response into our own wrapper object + response = self.ResponseCls.from_base( + base_response, pool=self, retries=retries, **response_kw + ) + # Everything went great! + clean_exit = True + except queue.Empty: + # Timed out by queue. + raise EmptyPoolError(self, "No pool connections are available.") + + except ( + TimeoutError, + SocketError, + ProtocolError, + h11.ProtocolError, + BaseSSLError, + SSLError, + CertificateError, + ) as e: + # Discard the connection for these exceptions. It will be + # replaced during the next _get_conn() call. + clean_exit = False + if isinstance(e, (BaseSSLError, CertificateError)): + e = SSLError(e) + elif isinstance( + e, (SocketError, NewConnectionError) + ) and self.proxy: + e = ProxyError('Cannot connect to proxy.', e) + elif isinstance(e, (SocketError, h11.ProtocolError)): + e = ProtocolError('Connection aborted.', e) + retries = retries.increment( + method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] + ) + retries.sleep() + # Keep track of the error for the retry warning. + err = e + finally: + if not clean_exit: + # We hit some kind of exception, handled or otherwise. We need + # to throw the connection away unless explicitly told not to. + # Close the connection, set the variable to None, and make sure + # we put the None back in the pool to avoid leaking it. + conn = conn and conn.close() + release_this_conn = True + if release_this_conn: + # Put the connection back to be reused. If the connection is + # expired then it will be None, which will get replaced with a + # fresh connection during _get_conn. + self._put_conn(conn) + if not conn: + # Try again + log.warning( + "Retrying (%r) after connection " "broken by '%r': %s", + retries, + err, + url, + ) + return self.urlopen( + method, + url, + body, + headers, + retries, + timeout=timeout, + pool_timeout=pool_timeout, + body_pos=body_pos, + **response_kw + ) + + def drain_and_release_conn(response): + try: + # discard any remaining response body, the connection will be + # released back to the pool once the entire response is read + response.read() + except ( + TimeoutError, + SocketError, + ProtocolError, + BaseSSLError, + SSLError, + ) as e: + pass + + # Check if we should retry the HTTP response. + has_retry_after = bool(response.getheader('Retry-After')) + if retries.is_retry(method, response.status, has_retry_after): + try: + retries = retries.increment( + method, url, response=response, _pool=self + ) + except MaxRetryError: + if retries.raise_on_status: + # Drain and release the connection for this response, since + # we're not returning it to be released manually. + drain_and_release_conn(response) + raise + + return response + + # drain and return the connection to the pool before recursing + drain_and_release_conn(response) + retries.sleep(response) + log.debug("Retry: %s", url) + return self.urlopen( + method, + url, + body, + headers, + retries=retries, + timeout=timeout, + pool_timeout=pool_timeout, + body_pos=body_pos, + **response_kw + ) + + return response + + +class HTTPSConnectionPool(HTTPConnectionPool): + """ + Same as :class:`.HTTPConnectionPool`, but HTTPS. + + When Python is compiled with the :mod:`ssl` module, then + :class:`.VerifiedHTTPSConnection` is used, which *can* verify certificates, + instead of :class:`.HTTPSConnection`. + + :class:`.VerifiedHTTPSConnection` uses one of ``assert_fingerprint``, + ``assert_hostname`` and ``host`` in this order to verify connections. + If ``assert_hostname`` is False, no verification is done. + + The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``, + ``ca_cert_dir``, and ``ssl_version`` are only used if :mod:`ssl` is + available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade + the connection socket into an SSL socket. + """ + scheme = 'https' + + def __init__( + self, + host, + port=None, + timeout=Timeout.DEFAULT_TIMEOUT, + maxsize=1, + block=False, + headers=None, + retries=None, + _proxy=None, + _proxy_headers=None, + key_file=None, + cert_file=None, + cert_reqs=None, + ca_certs=None, + ssl_version=None, + assert_hostname=None, + assert_fingerprint=None, + ca_cert_dir=None, + ssl_context=None, + **conn_kw + ): + HTTPConnectionPool.__init__( + self, + host, + port, + timeout, + maxsize, + block, + headers, + retries, + _proxy, + _proxy_headers, + **conn_kw + ) + if ssl is None: + raise SSLError("SSL module is not available") + + if ca_certs and cert_reqs is None: + cert_reqs = 'CERT_REQUIRED' + self.ssl_context = _build_context( + ssl_context, + keyfile=key_file, + certfile=cert_file, + cert_reqs=cert_reqs, + ca_certs=ca_certs, + ca_cert_dir=ca_cert_dir, + ssl_version=ssl_version, + ) + self.assert_hostname = assert_hostname + self.assert_fingerprint = assert_fingerprint + + def _new_conn(self): + """ + Return a fresh connection. + """ + self.num_connections += 1 + log.debug( + "Starting new HTTPS connection (%d): %s:%s", + self.num_connections, + self.host, + self.port or "443", + ) + actual_host = self.host + actual_port = self.port + tunnel_host = None + tunnel_port = None + tunnel_headers = None + if self.proxy is not None: + actual_host = self.proxy.host + actual_port = self.proxy.port + tunnel_host = self.host + tunnel_port = self.port + tunnel_headers = self.proxy_headers + + # TODO: Huge hack. + for kw in ('strict', 'redirect'): + if kw in self.conn_kw: + self.conn_kw.pop(kw) + + conn = self.ConnectionCls( + host=actual_host, + port=actual_port, + tunnel_host=tunnel_host, + tunnel_port=tunnel_port, + tunnel_headers=tunnel_headers, + ** self.conn_kw + ) + return conn + + def _start_conn(self, conn, connect_timeout): + """ + Called right before a request is made, after the socket is created. + """ + conn.connect( + ssl_context=self.ssl_context, + fingerprint=self.assert_fingerprint, + assert_hostname=self.assert_hostname, + connect_timeout=connect_timeout, + ) + if not conn.is_verified: + warnings.warn( + ( + 'Unverified HTTPS request is being made. ' + 'Adding certificate verification is strongly advised. See: ' + 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' + '#ssl-warnings' + ), + InsecureRequestWarning, + ) + + +def connection_from_url(url, **kw): + """ + Given a url, return an :class:`.ConnectionPool` instance of its host. + + This is a shortcut for not having to parse out the scheme, host, and port + of the url before creating an :class:`.ConnectionPool` instance. + + :param url: + Absolute URL string that must include the scheme. Port is optional. + + :param \\**kw: + Passes additional parameters to the constructor of the appropriate + :class:`.ConnectionPool`. Useful for specifying things like + timeout, maxsize, headers, etc. + + Example:: + + >>> conn = connection_from_url('http://google.com/') + >>> r = conn.request('GET', '/') + """ + scheme, host, port = get_host(url) + port = port or DEFAULT_PORTS.get(scheme, 80) + if scheme == 'https': + return HTTPSConnectionPool(host, port=port, **kw) + + else: + return HTTPConnectionPool(host, port=port, **kw) + + +def _ipv6_host(host): + """ + Process IPv6 address literals + """ + # httplib doesn't like it when we include brackets in IPv6 addresses + # Specifically, if we include brackets but also pass the port then + # httplib crazily doubles up the square brackets on the Host header. + # Instead, we need to make sure we never pass ``None`` as the port. + # However, for backward compatibility reasons we can't actually + # *assert* that. See http://bugs.python.org/issue28539 + # + # Also if an IPv6 address literal has a zone identifier, the + # percent sign might be URIencoded, convert it back into ASCII + if host.startswith('[') and host.endswith(']'): + host = host.replace('%25', '%').strip('[]') + return host diff --git a/requests3/core/_http/_sync/poolmanager.py b/requests3/core/_http/_sync/poolmanager.py new file mode 100644 index 00000000..9e0b4af1 --- /dev/null +++ b/requests3/core/_http/_sync/poolmanager.py @@ -0,0 +1,446 @@ +from __future__ import absolute_import +import collections +import functools +import logging + +from .._collections import RecentlyUsedContainer +from ..base import DEFAULT_PORTS +from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool +from ..exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown +from ..packages.six.moves.urllib.parse import urljoin +from ..request import RequestMethods +from ..util.url import parse_url +from ..util.request import set_file_position +from ..util.retry import Retry + +__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] +log = logging.getLogger(__name__) +SSL_KEYWORDS = ( + 'key_file', + 'cert_file', + 'cert_reqs', + 'ca_certs', + 'ssl_version', + 'ca_cert_dir', + 'ssl_context', +) +# All known keyword arguments that could be provided to the pool manager, its +# pools, or the underlying connections. This is used to construct a pool key. +_key_fields = ( + 'key_scheme', # str + 'key_host', # str + 'key_strict', + 'key_port', # int + 'key_timeout', # int or float or Timeout + 'key_retries', # int or Retry + 'key_block', # bool + 'key_source_address', # str + 'key_key_file', # str + 'key_cert_file', # str + 'key_cert_reqs', # str + 'key_ca_certs', # str + 'key_ssl_version', # str + 'key_ca_cert_dir', # str + 'key_ssl_context', # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext + 'key_maxsize', # int + 'key_headers', # dict + 'key__proxy', # parsed proxy url + 'key__proxy_headers', # dict + 'key_socket_options', # list of (level (int), optname (int), value (int or str)) tuples + 'key__socks_options', # dict + 'key_assert_hostname', # bool or string + 'key_assert_fingerprint', # str +) +# : The namedtuple class used to construct keys for the connection pool. +#: All custom key schemes should include the fields in this key at a minimum. +PoolKey = collections.namedtuple('PoolKey', _key_fields) + + +def _default_key_normalizer(key_class, request_context): + """ + Create a pool key out of a request context dictionary. + + According to RFC 3986, both the scheme and host are case-insensitive. + Therefore, this function normalizes both before constructing the pool + key for an HTTPS request. If you wish to change this behaviour, provide + alternate callables to ``key_fn_by_scheme``. + + :param key_class: + The class to use when constructing the key. This should be a namedtuple + with the ``scheme`` and ``host`` keys at a minimum. + :type key_class: namedtuple + :param request_context: + A dictionary-like object that contain the context for a request. + :type request_context: dict + + :return: A namedtuple that can be used as a connection pool key. + :rtype: PoolKey + """ + # Since we mutate the dictionary, make a copy first + context = request_context.copy() + context['scheme'] = context['scheme'].lower() + context['host'] = context['host'].lower() + # These are both dictionaries and need to be transformed into frozensets + for key in ('headers', '_proxy_headers', '_socks_options'): + if key in context and context[key] is not None: + context[key] = frozenset(context[key].items()) + # The socket_options key may be a list and needs to be transformed into a + # tuple. + socket_opts = context.get('socket_options') + if socket_opts is not None: + context['socket_options'] = tuple(socket_opts) + # Map the kwargs to the names in the namedtuple - this is necessary since + # namedtuples can't have fields starting with '_'. + for key in list(context.keys()): + context['key_' + key] = context.pop(key) + # Default to ``None`` for keys missing from the context + for field in key_class._fields: + if field not in context: + context[field] = None + return key_class(**context) + + +# : A dictionary that maps a scheme to a callable that creates a pool key. +#: This can be used to alter the way pool keys are constructed, if desired. +#: Each PoolManager makes a copy of this dictionary so they can be configured +#: globally here, or individually on the instance. +key_fn_by_scheme = { + 'http': functools.partial(_default_key_normalizer, PoolKey), + 'https': functools.partial(_default_key_normalizer, PoolKey), +} +pool_classes_by_scheme = { + 'http': HTTPConnectionPool, 'https': HTTPSConnectionPool +} + + +class PoolManager(RequestMethods): + """ + Allows for arbitrary requests while transparently keeping track of + necessary connection pools for you. + + :param num_pools: + Number of connection pools to cache before discarding the least + recently used pool. + + :param headers: + Headers to include with all requests, unless other headers are given + explicitly. + + :param \\**connection_pool_kw: + Additional parameters are used to create fresh + :class:`urllib3.connectionpool.ConnectionPool` instances. + + Example:: + + >>> manager = PoolManager(num_pools=2) + >>> r = manager.request('GET', 'http://google.com/') + >>> r = manager.request('GET', 'http://google.com/mail') + >>> r = manager.request('GET', 'http://yahoo.com/') + >>> len(manager.pools) + 2 + + """ + proxy = None + + def __init__( + self, num_pools=10, headers=None, backend=None, **connection_pool_kw + ): + RequestMethods.__init__(self, headers) + self.connection_pool_kw = connection_pool_kw + self.pools = RecentlyUsedContainer( + num_pools, dispose_func=lambda p: p.close() + ) + # Locally set the pool classes and keys so other PoolManagers can + # override them. + self.pool_classes_by_scheme = pool_classes_by_scheme + self.key_fn_by_scheme = key_fn_by_scheme.copy() + self.backend = backend + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.clear() + # Return False to re-raise any potential exceptions + return False + + def _new_pool(self, scheme, host, port, request_context=None): + """ + Create a new :class:`ConnectionPool` based on host, port, scheme, and + any additional pool keyword arguments. + + If ``request_context`` is provided, it is provided as keyword arguments + to the pool class used. This method is used to actually create the + connection pools handed out by :meth:`connection_from_url` and + companion methods. It is intended to be overridden for customization. + """ + pool_cls = self.pool_classes_by_scheme[scheme] + if request_context is None: + request_context = self.connection_pool_kw.copy() + # Although the context has everything necessary to create the pool, + # this function has historically only used the scheme, host, and port + # in the positional args. When an API change is acceptable these can + # be removed. + for key in ('scheme', 'host', 'port'): + request_context.pop(key, None) + if scheme == 'http': + for kw in SSL_KEYWORDS: + request_context.pop(kw, None) + return pool_cls(host, port, backend=self.backend, **request_context) + + def clear(self): + """ + Empty our store of pools and direct them all to close. + + This will not affect in-flight connections, but they will not be + re-used after completion. + """ + self.pools.clear() + + def connection_from_host( + self, host, port=None, scheme='http', pool_kwargs=None + ): + """ + Get a :class:`ConnectionPool` based on the host, port, and scheme. + + If ``port`` isn't given, it will be derived from the ``scheme`` using + ``urllib3.connectionpool.port_by_scheme``. If ``pool_kwargs`` is + provided, it is merged with the instance's ``connection_pool_kw`` + variable and used to create the new connection pool, if one is + needed. + """ + if not host: + raise LocationValueError("No host specified.") + + request_context = self._merge_pool_kwargs(pool_kwargs) + request_context['scheme'] = scheme or 'http' + if not port: + port = DEFAULT_PORTS.get(request_context['scheme'].lower(), 80) + request_context['port'] = port + request_context['host'] = host + return self.connection_from_context(request_context) + + def connection_from_context(self, request_context): + """ + Get a :class:`ConnectionPool` based on the request context. + + ``request_context`` must at least contain the ``scheme`` key and its + value must be a key in ``key_fn_by_scheme`` instance variable. + """ + scheme = request_context['scheme'].lower() + pool_key_constructor = self.key_fn_by_scheme[scheme] + pool_key = pool_key_constructor(request_context) + return self.connection_from_pool_key( + pool_key, request_context=request_context + ) + + def connection_from_pool_key(self, pool_key, request_context=None): + """ + Get a :class:`ConnectionPool` based on the provided pool key. + + ``pool_key`` should be a namedtuple that only contains immutable + objects. At a minimum it must have the ``scheme``, ``host``, and + ``port`` fields. + """ + with self.pools.lock: + # If the scheme, host, or port doesn't match existing open + # connections, open a new ConnectionPool. + pool = self.pools.get(pool_key) + if pool: + return pool + + # Make a fresh ConnectionPool of the desired type + scheme = request_context['scheme'] + host = request_context['host'] + port = request_context['port'] + pool = self._new_pool( + scheme, host, port, request_context=request_context + ) + self.pools[pool_key] = pool + return pool + + def connection_from_url(self, url, pool_kwargs=None): + """ + Similar to :func:`urllib3.connectionpool.connection_from_url`. + + If ``pool_kwargs`` is not provided and a new pool needs to be + constructed, ``self.connection_pool_kw`` is used to initialize + the :class:`urllib3.connectionpool.ConnectionPool`. If ``pool_kwargs`` + is provided, it is used instead. Note that if a new pool does not + need to be created for the request, the provided ``pool_kwargs`` are + not used. + """ + u = parse_url(url) + return self.connection_from_host( + u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs + ) + + def _merge_pool_kwargs(self, override): + """ + Merge a dictionary of override values for self.connection_pool_kw. + + This does not modify self.connection_pool_kw and returns a new dict. + Any keys in the override dictionary with a value of ``None`` are + removed from the merged dictionary. + """ + base_pool_kwargs = self.connection_pool_kw.copy() + if override: + for key, value in override.items(): + if value is None: + try: + del base_pool_kwargs[key] + except KeyError: + pass + else: + base_pool_kwargs[key] = value + return base_pool_kwargs + + def urlopen(self, method, url, redirect=True, **kw): + """ + Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen` + with redirect logic and only sends the request-uri portion of the + ``url``. + + The given ``url`` parameter must be absolute, such that an appropriate + :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it. + """ + u = parse_url(url) + conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme) + # Rewind body position, if needed. Record current position + # for future rewinds in the event of a redirect/retry. + body = kw.get('body') + body_pos = kw.get('body_pos') + kw['body_pos'] = set_file_position(body, body_pos) + if 'headers' not in kw: + kw['headers'] = self.headers + if self.proxy is not None and u.scheme == "http": + response = conn.urlopen(method, url, **kw) + else: + response = conn.urlopen(method, u.request_uri, **kw) + redirect_location = redirect and response.get_redirect_location() + if not redirect_location: + return response + + # Support relative URLs for redirecting. + redirect_location = urljoin(url, redirect_location) + # RFC 7231, Section 6.4.4 + if response.status == 303: + method = 'GET' + retries = kw.get('retries') + if not isinstance(retries, Retry): + retries = Retry.from_int(retries, redirect=redirect) + try: + retries = retries.increment( + method, url, response=response, _pool=conn + ) + except MaxRetryError: + if retries.raise_on_redirect: + raise + + return response + + kw['retries'] = retries + kw['redirect'] = redirect + retries.sleep_for_retry(response) + log.info("Redirecting %s -> %s", url, redirect_location) + return self.urlopen(method, redirect_location, **kw) + + +class ProxyManager(PoolManager): + """ + Behaves just like :class:`PoolManager`, but sends all requests through + the defined proxy, using the CONNECT method for HTTPS URLs. + + :param proxy_url: + The URL of the proxy to be used. + + :param proxy_headers: + A dictionary contaning headers that will be sent to the proxy. In case + of HTTP they are being sent with each request, while in the + HTTPS/CONNECT case they are sent only once. Could be used for proxy + authentication. + + Example: + >>> proxy = urllib3.ProxyManager('http://localhost:3128/') + >>> r1 = proxy.request('GET', 'http://google.com/') + >>> r2 = proxy.request('GET', 'http://httpbin.org/') + >>> len(proxy.pools) + 1 + >>> r3 = proxy.request('GET', 'https://httpbin.org/') + >>> r4 = proxy.request('GET', 'https://twitter.com/') + >>> len(proxy.pools) + 3 + + """ + + def __init__( + self, + proxy_url, + num_pools=10, + headers=None, + proxy_headers=None, + **connection_pool_kw + ): + if isinstance(proxy_url, HTTPConnectionPool): + proxy_url = '%s://%s:%i' % ( + proxy_url.scheme, proxy_url.host, proxy_url.port + ) + proxy = parse_url(proxy_url) + if not proxy.port: + port = DEFAULT_PORTS.get(proxy.scheme, 80) + proxy = proxy._replace(port=port) + if proxy.scheme not in ("http", "https"): + raise ProxySchemeUnknown(proxy.scheme) + + self.proxy = proxy + self.proxy_headers = proxy_headers or {} + connection_pool_kw['_proxy'] = self.proxy + connection_pool_kw['_proxy_headers'] = self.proxy_headers + super(ProxyManager, self).__init__( + num_pools, headers, **connection_pool_kw + ) + + def connection_from_host( + self, host, port=None, scheme='http', pool_kwargs=None + ): + if scheme == "https": + return super(ProxyManager, self).connection_from_host( + host, port, scheme, pool_kwargs=pool_kwargs + ) + + return super(ProxyManager, self).connection_from_host( + self.proxy.host, + self.proxy.port, + self.proxy.scheme, + pool_kwargs=pool_kwargs, + ) + + def _set_proxy_headers(self, url, headers=None): + """ + Sets headers needed by proxies: specifically, the Accept and Host + headers. Only sets headers not provided by the user. + """ + headers_ = {'Accept': '*/*'} + netloc = parse_url(url).netloc + if netloc: + headers_['Host'] = netloc + if headers: + headers_.update(headers) + return headers_ + + def urlopen(self, method, url, redirect=True, **kw): + "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." + u = parse_url(url) + if u.scheme == "http": + # For proxied HTTPS requests, httplib sets the necessary headers + # on the CONNECT to the proxy. For HTTP, we'll definitely + # need to set 'Host' at the very least. + headers = kw.get('headers', self.headers) + kw['headers'] = self._set_proxy_headers(url, headers) + return super(ProxyManager, self).urlopen( + method, url, redirect=redirect, **kw + ) + + +def proxy_from_url(url, **kw): + return ProxyManager(proxy_url=url, **kw) diff --git a/requests3/core/_http/_sync/response.py b/requests3/core/_http/_sync/response.py new file mode 100644 index 00000000..d3f59556 --- /dev/null +++ b/requests3/core/_http/_sync/response.py @@ -0,0 +1,461 @@ +from __future__ import absolute_import +from contextlib import contextmanager +import zlib +import io +import logging +from socket import timeout as SocketTimeout +from socket import error as SocketError + +import h11 + +from .._collections import HTTPHeaderDict +from ..exceptions import (ProtocolError, DecodeError, ReadTimeoutError) +from ..packages.six import string_types as basestring, binary_type +from ..util.ssl_ import BaseSSLError + +log = logging.getLogger(__name__) + + +class DeflateDecoder(object): + + def __init__(self): + self._first_try = True + self._data = binary_type() + self._obj = zlib.decompressobj() + + def __getattr__(self, name): + return getattr(self._obj, name) + + def decompress(self, data): + if not data: + return data + + if not self._first_try: + return self._obj.decompress(data) + + self._data += data + try: + decompressed = self._obj.decompress(data) + if decompressed: + self._first_try = False + self._data = None + return decompressed + + except zlib.error: + self._first_try = False + self._obj = zlib.decompressobj(-zlib.MAX_WBITS) + try: + return self.decompress(self._data) + + finally: + self._data = None + + +class GzipDecoder(object): + + def __init__(self): + self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) + + def __getattr__(self, name): + return getattr(self._obj, name) + + def decompress(self, data): + if not data: + return data + + return self._obj.decompress(data) + + +def _get_decoder(mode): + if mode == 'gzip': + return GzipDecoder() + + return DeflateDecoder() + + +class HTTPResponse(io.IOBase): + """ + HTTP Response container. + + Backwards-compatible to httplib's HTTPResponse but the response ``body`` is + loaded and decoded on-demand when the ``data`` property is accessed. This + class is also compatible with the Python standard library's :mod:`io` + module, and can hence be treated as a readable object in the context of that + framework. + + Extra parameters for behaviour not present in httplib.HTTPResponse: + + :param preload_content: + If True, the response's body will be preloaded during construction. + + :param decode_content: + If True, attempts to decode specific content-encoding's based on headers + (like 'gzip' and 'deflate') will be skipped and raw data will be used + instead. + + :param retries: + The retries contains the last :class:`~urllib3.util.retry.Retry` that + was used during the request. + """ + CONTENT_DECODERS = ['gzip', 'deflate'] + REDIRECT_STATUSES = [301, 302, 303, 307, 308] + + def __init__( + self, + body='', + headers=None, + status=0, + version=0, + reason=None, + strict=0, + preload_content=True, + decode_content=True, + original_response=None, + pool=None, + connection=None, + retries=None, + request_method=None, + ): + if isinstance(headers, HTTPHeaderDict): + self.headers = headers + else: + self.headers = HTTPHeaderDict(headers) + self.status = status + self.version = version + self.reason = reason + self.strict = strict + self.decode_content = decode_content + self.retries = retries + self._decoder = None + self._body = None + self._fp = None + self._original_response = original_response + self._fp_bytes_read = 0 + self._buffer = b'' + if body and isinstance(body, (basestring, binary_type)): + self._body = body + else: + self._fp = body + self._pool = pool + self._connection = connection + # If requested, preload the body. + if preload_content and not self._body: + self._body = self.read(decode_content=decode_content) + + def get_redirect_location(self): + """ + Should we redirect and where to? + + :returns: Truthy redirect location string if we got a redirect status + code and valid location. ``None`` if redirect status and no + location. ``False`` if not a redirect status code. + """ + if self.status in self.REDIRECT_STATUSES: + return self.headers.get('location') + + return False + + def release_conn(self): + if not self._pool or not self._connection: + return + + self._pool._put_conn(self._connection) + self._connection = None + + @property + def data(self): + # For backwords-compat with earlier urllib3 0.4 and earlier. + if self._body is not None: + return self._body + + if self._fp: + return self.read(cache_content=True) + + @property + def connection(self): + return self._connection + + def tell(self): + """ + Obtain the number of bytes pulled over the wire so far. May differ from + the amount of content returned by :meth:``HTTPResponse.read`` if bytes + are encoded on the wire (e.g, compressed). + """ + return self._fp_bytes_read + + def _init_decoder(self): + """ + Set-up the _decoder attribute if necessary. + """ + # Note: content-encoding value should be case-insensitive, per RFC 7230 + # Section 3.2 + content_encoding = self.headers.get('content-encoding', '').lower() + if self._decoder is None and content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) + + def _decode(self, data, decode_content, flush_decoder): + """ + Decode the data passed in and potentially flush the decoder. + """ + try: + if decode_content and self._decoder: + data = self._decoder.decompress(data) + except (IOError, zlib.error) as e: + content_encoding = self.headers.get('content-encoding', '').lower() + raise DecodeError( + "Received response with content-encoding: %s, but " + "failed to decode it." % content_encoding, + e, + ) + + if flush_decoder and decode_content: + data += self._flush_decoder() + return data + + def _flush_decoder(self): + """ + Flushes the decoder. Should only be called if the decoder is actually + being used. + """ + if self._decoder: + buf = self._decoder.decompress(b'') + return buf + self._decoder.flush() + + return b'' + + @contextmanager + def _error_catcher(self): + """ + Catch low-level python exceptions, instead re-raising urllib3 + variants, so that low-level exceptions are not leaked in the + high-level api. + + On exit, release the connection back to the pool. + """ + clean_exit = False + try: + try: + yield + + except SocketTimeout: + # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but + # there is yet no clean way to get at it from this context. + raise ReadTimeoutError(self._pool, None, 'Read timed out.') + + except BaseSSLError as e: + # FIXME: Is there a better way to differentiate between SSLErrors? + if 'read operation timed out' not in str(e): # Defensive: + # This shouldn't happen but just in case we're missing an edge + # case, let's avoid swallowing SSL errors. + raise + + raise ReadTimeoutError(self._pool, None, 'Read timed out.') + + except (h11.ProtocolError, SocketError) as e: + # This includes IncompleteRead. + raise ProtocolError('Connection broken: %r' % e, e) + + except GeneratorExit: + # We swallow GeneratorExit when it is emitted: this allows the + # use of the error checker inside stream() + pass + # If no exception is thrown, we should avoid cleaning up + # unnecessarily. + clean_exit = True + finally: + # If we didn't terminate cleanly, we need to throw away our + # connection. + if not clean_exit: + self.close() + # If we hold the original response but it's finished now, we should + # return the connection back to the pool. + # XXX + if False and self._original_response and self._original_response.complete: + self.release_conn() + + def read(self, amt=None, decode_content=None, cache_content=False): + """ + Similar to :meth:`httplib.HTTPResponse.read`, but with two additional + parameters: ``decode_content`` and ``cache_content``. + + :param amt: + How much of the content to read. If specified, caching is skipped + because it doesn't make sense to cache partial content as the full + response. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + + :param cache_content: + If True, will save the returned data such that the same result is + returned despite of the state of the underlying file object. This + is useful if you want the ``.data`` property to continue working + after having ``.read()`` the file object. (Overridden if ``amt`` is + set.) + """ + # TODO: refactor this method to better handle buffered output. + # This method is a weird one. We treat this read() like a buffered + # read, meaning that it never reads "short" unless there is an EOF + # condition at work. However, we have a decompressor in play here, + # which means our read() returns decompressed data. + # + # This means the buffer can only meaningfully buffer decompressed data. + # This makes this method prone to over-reading, and forcing too much + # data into the buffer. That's unfortunate, but right now I'm not smart + # enough to come up with a way to solve that problem. + if self._fp is None and not self._buffer: + return b'' + + data = self._buffer + with self._error_catcher(): + if amt is None: + chunks = [] + for chunk in self.stream(decode_content): + chunks.append(chunk) + data += b''.join(chunks) + self._buffer = b'' + # We only cache the body data for simple read calls. + self._body = data + else: + data_len = len(data) + chunks = [data] + streamer = self.stream(decode_content) + while data_len < amt: + try: + chunk = next(streamer) + except StopIteration: + break + + else: + chunks.append(chunk) + data_len += len(chunk) + data = b''.join(chunks) + self._buffer = data[amt:] + data = data[:amt] + return data + + def stream(self, decode_content=None): + """ + A generator wrapper for the read() method. + + :param decode_content: + If True, will attempt to decode the body based on the + 'content-encoding' header. + """ + # Short-circuit evaluation for exhausted responses. + if self._fp is None: + return + + self._init_decoder() + if decode_content is None: + decode_content = self.decode_content + with self._error_catcher(): + for raw_chunk in self._fp: + self._fp_bytes_read += len(raw_chunk) + decoded_chunk = self._decode( + raw_chunk, decode_content, flush_decoder=False + ) + if decoded_chunk: + yield decoded_chunk + + # This branch is speculative: most decoders do not need to flush, + # and so this produces no output. However, it's here because + # anecdotally some platforms on which we do not test (like Jython) + # do require the flush. For this reason, we exclude this from code + # coverage. Happily, the code here is so simple that testing the + # branch we don't enter is basically entirely unnecessary (it's + # just a yield statement). + final_chunk = self._decode(b'', decode_content, flush_decoder=True) + if final_chunk: # Platform-specific: Jython + yield final_chunk + + self._fp = None + + @classmethod + def from_base(ResponseCls, r, **response_kw): + """ + Given an :class:`urllib3.base.Response` instance ``r``, return a + corresponding :class:`urllib3.response.HTTPResponse` object. + + Remaining parameters are passed to the HTTPResponse constructor, along + with ``original_response=r``. + """ + # TODO: Huge hack. + for kw in ('redirect', 'assert_same_host', 'enforce_content_length'): + if kw in response_kw: + response_kw.pop(kw) + + resp = ResponseCls( + body=r.body, + headers=r.headers, + status=r.status_code, + version=r.version, + original_response=r, + connection=r.body, + **response_kw + ) + return resp + + + # Backwards-compatibility methods for httplib.HTTPResponse + def getheaders(self): + return self.headers + + def getheader(self, name, default=None): + return self.headers.get(name, default) + + + # Backwards compatibility for http.cookiejar + def info(self): + return self.headers + + + # Overrides from io.IOBase + def close(self): + if not self.closed: + self._fp.close() + self._buffer = b'' + self._fp = None + if self._connection: + self._connection.close() + + @property + def closed(self): + # This method is required for `io` module compatibility. + if self._fp is None and not self._buffer: + return True + + elif hasattr(self._fp, 'complete'): + return self._fp.complete + + else: + return False + + def fileno(self): + # This method is required for `io` module compatibility. + if self._fp is None: + raise IOError("HTTPResponse has no file to get a fileno from") + + elif hasattr(self._fp, "fileno"): + return self._fp.fileno() + + else: + raise IOError( + "The file-like object this HTTPResponse is wrapped " + "around has no file descriptor" + ) + + def readable(self): + # This method is required for `io` module compatibility. + return True + + def readinto(self, b): + # This method is required for `io` module compatibility. + temp = self.read(len(b)) + if len(temp) == 0: + return 0 + + else: + b[:len(temp)] = temp + return len(temp) diff --git a/requests3/core/_http/base.py b/requests3/core/_http/base.py new file mode 100644 index 00000000..1dbe94a6 --- /dev/null +++ b/requests3/core/_http/base.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +""" +This module provides the base structure of the Request/Response objects that +urllib3 passes around to manage its HTTP semantic layer. + +These objects are the lowest common denominator: that is, they define the +Request/Response functionality that is always supported by urllib3. This means +they do not include any extra function required for asynchrony: that +functionality is handled elsewhere. Any part of urllib3 is required to be able +to work with one of these objects. +""" +from ._collections import HTTPHeaderDict + +# This dictionary is used to store the default ports for specific schemes to +# control whether the port is inserted into the Host header. +DEFAULT_PORTS = {"http": 80, "https": 443} + + +class Request(object): + """ + The base, common, Request object. + + This object provides a *semantic* representation of a HTTP request. It + includes all the magical parts of a HTTP request that we have come to know + and love: it has a method, a target (the path & query portions of a URI), + some headers, and optionally a body. + + All of urllib3 manipulates these Request objects, passing them around and + changing them as necessary. The low-level layers know how to send these + objects. + """ + + def __init__(self, method, target, headers=None, body=None): + # : The HTTP method in use. Must be a byte string. + self.method = method + # : The request target: that is, the path and query portions of the URI. + self.target = target + # : The request headers. These are always stored as a HTTPHeaderDict. + self.headers = HTTPHeaderDict(headers) + # : The request body. This is allowed to be one a few kind of objects: + #: - A byte string. + #: - A "readable" object. + #: - An iterable of byte strings. + #: - A text string (not recommended, auto-encoded to UTF-8) + self.body = body + + def add_host(self, host, port, scheme): + """ + Add the Host header, as needed. + + This helper method exists to circumvent an ordering problem: the best + layer to add the Host header is the bottom layer, but it is the layer + that will add headers last. That means that they will appear at the + bottom of the header block. + + Proxies, caches, and other intermediaries *hate* it when clients do + that because the Host header is routing information, and they'd like to + see it as early as possible. For this reason, this method ensures that + the Host header will be the first one emitted. It also ensures that we + do not duplicate the host header: if there already is one, we just use + that one. + """ + if b'host' not in self.headers: + # We test against a sentinel object here to forcibly always insert + # the port for schemes we don't understand. + if port is DEFAULT_PORTS.get(scheme, object()): + header = host + else: + header = "{}:{}".format(host, port) + headers = HTTPHeaderDict(host=header) + headers._copy_from(self.headers) + self.headers = headers + + +class Response(object): + """ + The abstract low-level Response object that urllib3 works on. This is not + the high-level helpful Response object that is exposed at the higher layers + of urllib3: it's just a simple object that just exposes the lowest-level + HTTP semantics to allow processing by the higher levels. + """ + + def __init__(self, status_code, headers, body, version): + # : The HTTP status code of the response. + self.status_code = status_code + # : The headers on the response, as a HTTPHeaderDict. + self.headers = HTTPHeaderDict(headers) + # : The request body. This is an iterable of bytes, and *must* be + #: iterated if the connection is to be preserved. + self.body = body + # : The HTTP version of the response. Stored as a bytestring. + self.version = version + + @property + def complete(self): + """ + If the response can be safely returned to the connection pool, returns + True. + """ + return self.body.complete diff --git a/requests3/core/_http/connection.py b/requests3/core/_http/connection.py new file mode 100644 index 00000000..14989de4 --- /dev/null +++ b/requests3/core/_http/connection.py @@ -0,0 +1,406 @@ +from __future__ import absolute_import +import datetime +import logging +import os +import sys +import socket +from socket import error as SocketError, timeout as SocketTimeout +import warnings +from .packages import six +from .packages.six.moves.http_client import HTTPConnection as _HTTPConnection +from .packages.six.moves.http_client import HTTPException # noqa: F401 + +try: # Compiled with SSL? + import ssl + + BaseSSLError = ssl.SSLError +except (ImportError, AttributeError): # Platform-specific: No SSL. + ssl = None + + class BaseSSLError(BaseException): + pass + + +try: # Python 3: + # Not a no-op, we're adding this to the namespace so it can be imported. + ConnectionError = ConnectionError +except NameError: # Python 2: + + class ConnectionError(Exception): + pass + + +from .exceptions import ( + NewConnectionError, + ConnectTimeoutError, + SubjectAltNameWarning, + SystemTimeWarning, +) +from .packages.ssl_match_hostname import match_hostname, CertificateError + +from .util.ssl_ import ( + resolve_cert_reqs, + resolve_ssl_version, + assert_fingerprint, + create_urllib3_context, + ssl_wrap_socket, +) + + +from .util import connection + +from ._collections import HTTPHeaderDict + +log = logging.getLogger(__name__) +port_by_scheme = {'http': 80, 'https': 443} +# When updating RECENT_DATE, move it to within two years of the current date, +# and not less than 6 months ago. +# Example: if Today is 2018-01-01, then RECENT_DATE should be any date on or +# after 2016-01-01 (today - 2 years) AND before 2017-07-01 (today - 6 months) +RECENT_DATE = datetime.date(2017, 6, 30) + + +class DummyConnection(object): + """Used to detect a failed ConnectionCls import.""" + pass + + +class HTTPConnection(_HTTPConnection, object): + """ + Based on httplib.HTTPConnection but provides an extra constructor + backwards-compatibility layer between older and newer Pythons. + + Additional keyword parameters are used to configure attributes of the connection. + Accepted parameters include: + + - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool` + - ``source_address``: Set the source address for the current connection. + + .. note:: This is ignored for Python 2.6. It is only applied for 2.7 and 3.x + + - ``socket_options``: Set specific options on the underlying socket. If not specified, then + defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling + Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy. + + For example, if you wish to enable TCP Keep Alive in addition to the defaults, + you might pass:: + + HTTPConnection.default_socket_options + [ + (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), + ] + + Or you may want to disable the defaults by passing an empty list (e.g., ``[]``). + """ + default_port = port_by_scheme['http'] + # : Disable Nagle's algorithm by default. + #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]`` + default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)] + # : Whether this connection verifies the host's certificate. + is_verified = False + + def __init__(self, *args, **kw): + if six.PY3: # Python 3 + kw.pop('strict', None) + # Pre-set source_address in case we have an older Python like 2.6. + self.source_address = kw.get('source_address') + if sys.version_info < (2, 7): # Python 2.6 + # _HTTPConnection on Python 2.6 will balk at this keyword arg, but + # not newer versions. We can still use it when creating a + # connection though, so we pop it *after* we have saved it as + # self.source_address. + kw.pop('source_address', None) + # : The socket options provided by the user. If no options are + #: provided, we use the default options. + self.socket_options = kw.pop( + 'socket_options', self.default_socket_options + ) + # Superclass also sets self.source_address in Python 2.7+. + _HTTPConnection.__init__(self, *args, **kw) + + @property + def host(self): + """ + Getter method to remove any trailing dots that indicate the hostname is an FQDN. + + In general, SSL certificates don't include the trailing dot indicating a + fully-qualified domain name, and thus, they don't validate properly when + checked against a domain name that includes the dot. In addition, some + servers may not expect to receive the trailing dot when provided. + + However, the hostname with trailing dot is critical to DNS resolution; doing a + lookup with the trailing dot will properly only resolve the appropriate FQDN, + whereas a lookup without a trailing dot will search the system's search domain + list. Thus, it's important to keep the original host around for use only in + those cases where it's appropriate (i.e., when doing DNS lookup to establish the + actual TCP connection across which we're going to send HTTP requests). + """ + return self._dns_host.rstrip('.') + + @host.setter + def host(self, value): + """ + Setter for the `host` property. + + We assume that only urllib3 uses the _dns_host attribute; httplib itself + only uses `host`, and it seems reasonable that other libraries follow suit. + """ + self._dns_host = value + + def _new_conn(self): + """ Establish a socket connection and set nodelay settings on it. + + :return: New socket connection. + """ + extra_kw = {} + if self.source_address: + extra_kw['source_address'] = self.source_address + if self.socket_options: + extra_kw['socket_options'] = self.socket_options + try: + conn = connection.create_connection( + (self._dns_host, self.port), self.timeout, **extra_kw + ) + except SocketTimeout as e: + raise ConnectTimeoutError( + self, + "Connection to %s timed out. (connect timeout=%s)" % + (self.host, self.timeout), + ) + + except SocketError as e: + raise NewConnectionError( + self, "Failed to establish a new connection: %s" % e + ) + + return conn + + def _prepare_conn(self, conn): + self.sock = conn + # the _tunnel_host attribute was added in python 2.6.3 (via + # http://hg.python.org/cpython/rev/0f57b30a152f) so pythons 2.6(0-2) do + # not have them. + if getattr(self, '_tunnel_host', None): + # TODO: Fix tunnel so it doesn't depend on self.sock state. + self._tunnel() + # Mark this connection as not reusable + self.auto_open = 0 + + def connect(self): + conn = self._new_conn() + self._prepare_conn(conn) + + def request_chunked(self, method, url, body=None, headers=None): + """ + Alternative to the common request method, which sends the + body with chunked encoding and not as one block + """ + headers = HTTPHeaderDict(headers if headers is not None else {}) + skip_accept_encoding = 'accept-encoding' in headers + skip_host = 'host' in headers + self.putrequest( + method, + url, + skip_accept_encoding=skip_accept_encoding, + skip_host=skip_host, + ) + for header, value in headers.items(): + self.putheader(header, value) + if 'transfer-encoding' not in headers: + self.putheader('Transfer-Encoding', 'chunked') + self.endheaders() + if body is not None: + stringish_types = six.string_types + (six.binary_type,) + if isinstance(body, stringish_types): + body = (body,) + for chunk in body: + if not chunk: + continue + + if not isinstance(chunk, six.binary_type): + chunk = chunk.encode('utf8') + len_str = hex(len(chunk))[2:] + self.send(len_str.encode('utf-8')) + self.send(b'\r\n') + self.send(chunk) + self.send(b'\r\n') + # After the if clause, to always have a closed body + self.send(b'0\r\n\r\n') + + +class HTTPSConnection(HTTPConnection): + default_port = port_by_scheme['https'] + ssl_version = None + + def __init__( + self, + host, + port=None, + key_file=None, + cert_file=None, + strict=None, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + ssl_context=None, + **kw + ): + HTTPConnection.__init__( + self, host, port, strict=strict, timeout=timeout, **kw + ) + self.key_file = key_file + self.cert_file = cert_file + self.ssl_context = ssl_context + # Required property for Google AppEngine 1.9.0 which otherwise causes + # HTTPS requests to go out as HTTP. (See Issue #356) + self._protocol = 'https' + + def connect(self): + conn = self._new_conn() + self._prepare_conn(conn) + if self.ssl_context is None: + self.ssl_context = create_urllib3_context( + ssl_version=resolve_ssl_version(None), + cert_reqs=resolve_cert_reqs(None), + ) + self.sock = ssl_wrap_socket( + sock=conn, + keyfile=self.key_file, + certfile=self.cert_file, + ssl_context=self.ssl_context, + ) + + +class VerifiedHTTPSConnection(HTTPSConnection): + """ + Based on httplib.HTTPSConnection but wraps the socket with + SSL certification. + """ + cert_reqs = None + ca_certs = None + ca_cert_dir = None + ssl_version = None + assert_fingerprint = None + + def set_cert( + self, + key_file=None, + cert_file=None, + cert_reqs=None, + ca_certs=None, + assert_hostname=None, + assert_fingerprint=None, + ca_cert_dir=None, + ): + """ + This method should only be called once, before the connection is used. + """ + # If cert_reqs is not provided, we can try to guess. If the user gave + # us a cert database, we assume they want to use it: otherwise, if + # they gave us an SSL Context object we should use whatever is set for + # it. + if cert_reqs is None: + if ca_certs or ca_cert_dir: + cert_reqs = 'CERT_REQUIRED' + elif self.ssl_context is not None: + cert_reqs = self.ssl_context.verify_mode + self.key_file = key_file + self.cert_file = cert_file + self.cert_reqs = cert_reqs + self.assert_hostname = assert_hostname + self.assert_fingerprint = assert_fingerprint + self.ca_certs = ca_certs and os.path.expanduser(ca_certs) + self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir) + + def connect(self): + # Add certificate verification + conn = self._new_conn() + hostname = self.host + if getattr(self, '_tunnel_host', None): + # _tunnel_host was added in Python 2.6.3 + # (See: http://hg.python.org/cpython/rev/0f57b30a152f) + self.sock = conn + # Calls self._set_hostport(), so self.host is + # self._tunnel_host below. + self._tunnel() + # Mark this connection as not reusable + self.auto_open = 0 + # Override the host with the one we're requesting data from. + hostname = self._tunnel_host + is_time_off = datetime.date.today() < RECENT_DATE + if is_time_off: + warnings.warn( + ( + 'System time is way off (before {0}). This will probably ' + 'lead to SSL verification errors' + ).format( + RECENT_DATE + ), + SystemTimeWarning, + ) + # Wrap socket using verification with the root certs in + # trusted_root_certs + if self.ssl_context is None: + self.ssl_context = create_urllib3_context( + ssl_version=resolve_ssl_version(self.ssl_version), + cert_reqs=resolve_cert_reqs(self.cert_reqs), + ) + context = self.ssl_context + context.verify_mode = resolve_cert_reqs(self.cert_reqs) + self.sock = ssl_wrap_socket( + sock=conn, + keyfile=self.key_file, + certfile=self.cert_file, + ca_certs=self.ca_certs, + ca_cert_dir=self.ca_cert_dir, + server_hostname=hostname, + ssl_context=context, + ) + if self.assert_fingerprint: + assert_fingerprint( + self.sock.getpeercert(binary_form=True), + self.assert_fingerprint, + ) + elif context.verify_mode != ssl.CERT_NONE and not getattr( + context, 'check_hostname', False + ) and self.assert_hostname is not False: + # While urllib3 attempts to always turn off hostname matching from + # the TLS library, this cannot always be done. So we check whether + # the TLS Library still thinks it's matching hostnames. + cert = self.sock.getpeercert() + if not cert.get('subjectAltName', ()): + warnings.warn( + ( + 'Certificate for {0} has no `subjectAltName`, falling back to check for a ' + '`commonName` for now. This feature is being removed by major browsers and ' + 'deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 ' + 'for details.)'.format(hostname) + ), + SubjectAltNameWarning, + ) + _match_hostname(cert, self.assert_hostname or hostname) + self.is_verified = ( + context.verify_mode == ssl.CERT_REQUIRED or + self.assert_fingerprint is not None + ) + + +def _match_hostname(cert, asserted_hostname): + try: + match_hostname(cert, asserted_hostname) + except CertificateError as e: + log.error( + 'Certificate did not match expected hostname: %s. ' + 'Certificate: %s', + asserted_hostname, + cert, + ) + # Add cert to exception and reraise so client code can inspect + # the cert when catching the exception, if they want to + e._peer_cert = cert + raise + + +if ssl: + # Make a copy for testing. + UnverifiedHTTPSConnection = HTTPSConnection + HTTPSConnection = VerifiedHTTPSConnection +else: + HTTPSConnection = DummyConnection diff --git a/requests3/core/_http/connectionpool.py b/requests3/core/_http/connectionpool.py new file mode 100644 index 00000000..7705c4d3 --- /dev/null +++ b/requests3/core/_http/connectionpool.py @@ -0,0 +1,13 @@ +from ._sync.connectionpool import ( + ConnectionPool, + HTTPConnectionPool, + HTTPSConnectionPool, + connection_from_url, +) + +__all__ = [ + 'ConnectionPool', + 'HTTPConnectionPool', + 'HTTPSConnectionPool', + 'connection_from_url', +] diff --git a/requests3/core/_http/contrib/__init__.py b/requests3/core/_http/contrib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/requests3/core/_http/contrib/_securetransport/__init__.py b/requests3/core/_http/contrib/_securetransport/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/requests3/core/_http/contrib/_securetransport/bindings.py b/requests3/core/_http/contrib/_securetransport/bindings.py new file mode 100644 index 00000000..fbba2915 --- /dev/null +++ b/requests3/core/_http/contrib/_securetransport/bindings.py @@ -0,0 +1,417 @@ +""" +This module uses ctypes to bind a whole bunch of functions and constants from +SecureTransport. The goal here is to provide the low-level API to +SecureTransport. These are essentially the C-level functions and constants, and +they're pretty gross to work with. + +This code is a bastardised version of the code found in Will Bond's oscrypto +library. An enormous debt is owed to him for blazing this trail for us. For +that reason, this code should be considered to be covered both by urllib3's +license and by oscrypto's: + + Copyright (c) 2015-2016 Will Bond + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +""" +from __future__ import absolute_import + +import platform +from ctypes.util import find_library +from ctypes import ( + c_void_p, + c_int32, + c_char_p, + c_size_t, + c_byte, + c_uint32, + c_ulong, + c_long, + c_bool, +) +from ctypes import CDLL, POINTER, CFUNCTYPE + +security_path = find_library('Security') +if not security_path: + raise ImportError('The library Security could not be found') + +core_foundation_path = find_library('CoreFoundation') +if not core_foundation_path: + raise ImportError('The library CoreFoundation could not be found') + +version = platform.mac_ver()[0] +version_info = tuple(map(int, version.split('.'))) +if version_info < (10, 8): + raise OSError( + 'Only OS X 10.8 and newer are supported, not %s.%s' % + (version_info[0], version_info[1]) + ) + +Security = CDLL(security_path, use_errno=True) +CoreFoundation = CDLL(core_foundation_path, use_errno=True) +Boolean = c_bool +CFIndex = c_long +CFStringEncoding = c_uint32 +CFData = c_void_p +CFString = c_void_p +CFArray = c_void_p +CFMutableArray = c_void_p +CFDictionary = c_void_p +CFError = c_void_p +CFType = c_void_p +CFTypeID = c_ulong +CFTypeRef = POINTER(CFType) +CFAllocatorRef = c_void_p +OSStatus = c_int32 +CFDataRef = POINTER(CFData) +CFStringRef = POINTER(CFString) +CFArrayRef = POINTER(CFArray) +CFMutableArrayRef = POINTER(CFMutableArray) +CFDictionaryRef = POINTER(CFDictionary) +CFArrayCallBacks = c_void_p +CFDictionaryKeyCallBacks = c_void_p +CFDictionaryValueCallBacks = c_void_p +SecCertificateRef = POINTER(c_void_p) +SecExternalFormat = c_uint32 +SecExternalItemType = c_uint32 +SecIdentityRef = POINTER(c_void_p) +SecItemImportExportFlags = c_uint32 +SecItemImportExportKeyParameters = c_void_p +SecKeychainRef = POINTER(c_void_p) +SSLProtocol = c_uint32 +SSLCipherSuite = c_uint32 +SSLContextRef = POINTER(c_void_p) +SecTrustRef = POINTER(c_void_p) +SSLConnectionRef = c_uint32 +SecTrustResultType = c_uint32 +SecTrustOptionFlags = c_uint32 +SSLProtocolSide = c_uint32 +SSLConnectionType = c_uint32 +SSLSessionOption = c_uint32 +try: + Security.SecItemImport.argtypes = [ + CFDataRef, + CFStringRef, + POINTER(SecExternalFormat), + POINTER(SecExternalItemType), + SecItemImportExportFlags, + POINTER(SecItemImportExportKeyParameters), + SecKeychainRef, + POINTER(CFArrayRef), + ] + Security.SecItemImport.restype = OSStatus + Security.SecCertificateGetTypeID.argtypes = [] + Security.SecCertificateGetTypeID.restype = CFTypeID + Security.SecIdentityGetTypeID.argtypes = [] + Security.SecIdentityGetTypeID.restype = CFTypeID + Security.SecKeyGetTypeID.argtypes = [] + Security.SecKeyGetTypeID.restype = CFTypeID + Security.SecCertificateCreateWithData.argtypes = [ + CFAllocatorRef, CFDataRef + ] + Security.SecCertificateCreateWithData.restype = SecCertificateRef + Security.SecCertificateCopyData.argtypes = [SecCertificateRef] + Security.SecCertificateCopyData.restype = CFDataRef + Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p] + Security.SecCopyErrorMessageString.restype = CFStringRef + Security.SecIdentityCreateWithCertificate.argtypes = [ + CFTypeRef, SecCertificateRef, POINTER(SecIdentityRef) + ] + Security.SecIdentityCreateWithCertificate.restype = OSStatus + Security.SecKeychainCreate.argtypes = [ + c_char_p, + c_uint32, + c_void_p, + Boolean, + c_void_p, + POINTER(SecKeychainRef), + ] + Security.SecKeychainCreate.restype = OSStatus + Security.SecKeychainDelete.argtypes = [SecKeychainRef] + Security.SecKeychainDelete.restype = OSStatus + Security.SecPKCS12Import.argtypes = [ + CFDataRef, CFDictionaryRef, POINTER(CFArrayRef) + ] + Security.SecPKCS12Import.restype = OSStatus + SSLReadFunc = CFUNCTYPE( + OSStatus, SSLConnectionRef, c_void_p, POINTER(c_size_t) + ) + SSLWriteFunc = CFUNCTYPE( + OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t) + ) + Security.SSLSetIOFuncs.argtypes = [ + SSLContextRef, SSLReadFunc, SSLWriteFunc + ] + Security.SSLSetIOFuncs.restype = OSStatus + Security.SSLSetPeerID.argtypes = [SSLContextRef, c_char_p, c_size_t] + Security.SSLSetPeerID.restype = OSStatus + Security.SSLSetCertificate.argtypes = [SSLContextRef, CFArrayRef] + Security.SSLSetCertificate.restype = OSStatus + Security.SSLSetCertificateAuthorities.argtypes = [ + SSLContextRef, CFTypeRef, Boolean + ] + Security.SSLSetCertificateAuthorities.restype = OSStatus + Security.SSLSetConnection.argtypes = [SSLContextRef, SSLConnectionRef] + Security.SSLSetConnection.restype = OSStatus + Security.SSLSetPeerDomainName.argtypes = [ + SSLContextRef, c_char_p, c_size_t + ] + Security.SSLSetPeerDomainName.restype = OSStatus + Security.SSLHandshake.argtypes = [SSLContextRef] + Security.SSLHandshake.restype = OSStatus + Security.SSLRead.argtypes = [ + SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t) + ] + Security.SSLRead.restype = OSStatus + Security.SSLWrite.argtypes = [ + SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t) + ] + Security.SSLWrite.restype = OSStatus + Security.SSLClose.argtypes = [SSLContextRef] + Security.SSLClose.restype = OSStatus + Security.SSLGetNumberSupportedCiphers.argtypes = [ + SSLContextRef, POINTER(c_size_t) + ] + Security.SSLGetNumberSupportedCiphers.restype = OSStatus + Security.SSLGetSupportedCiphers.argtypes = [ + SSLContextRef, POINTER(SSLCipherSuite), POINTER(c_size_t) + ] + Security.SSLGetSupportedCiphers.restype = OSStatus + Security.SSLSetEnabledCiphers.argtypes = [ + SSLContextRef, POINTER(SSLCipherSuite), c_size_t + ] + Security.SSLSetEnabledCiphers.restype = OSStatus + Security.SSLGetNumberEnabledCiphers.argtype = [ + SSLContextRef, POINTER(c_size_t) + ] + Security.SSLGetNumberEnabledCiphers.restype = OSStatus + Security.SSLGetEnabledCiphers.argtypes = [ + SSLContextRef, POINTER(SSLCipherSuite), POINTER(c_size_t) + ] + Security.SSLGetEnabledCiphers.restype = OSStatus + Security.SSLGetNegotiatedCipher.argtypes = [ + SSLContextRef, POINTER(SSLCipherSuite) + ] + Security.SSLGetNegotiatedCipher.restype = OSStatus + Security.SSLGetNegotiatedProtocolVersion.argtypes = [ + SSLContextRef, POINTER(SSLProtocol) + ] + Security.SSLGetNegotiatedProtocolVersion.restype = OSStatus + Security.SSLCopyPeerTrust.argtypes = [SSLContextRef, POINTER(SecTrustRef)] + Security.SSLCopyPeerTrust.restype = OSStatus + Security.SecTrustSetAnchorCertificates.argtypes = [SecTrustRef, CFArrayRef] + Security.SecTrustSetAnchorCertificates.restype = OSStatus + Security.SecTrustSetAnchorCertificatesOnly.argstypes = [ + SecTrustRef, Boolean + ] + Security.SecTrustSetAnchorCertificatesOnly.restype = OSStatus + Security.SecTrustEvaluate.argtypes = [ + SecTrustRef, POINTER(SecTrustResultType) + ] + Security.SecTrustEvaluate.restype = OSStatus + Security.SecTrustGetCertificateCount.argtypes = [SecTrustRef] + Security.SecTrustGetCertificateCount.restype = CFIndex + Security.SecTrustGetCertificateAtIndex.argtypes = [SecTrustRef, CFIndex] + Security.SecTrustGetCertificateAtIndex.restype = SecCertificateRef + Security.SSLCreateContext.argtypes = [ + CFAllocatorRef, SSLProtocolSide, SSLConnectionType + ] + Security.SSLCreateContext.restype = SSLContextRef + Security.SSLSetSessionOption.argtypes = [ + SSLContextRef, SSLSessionOption, Boolean + ] + Security.SSLSetSessionOption.restype = OSStatus + Security.SSLSetProtocolVersionMin.argtypes = [SSLContextRef, SSLProtocol] + Security.SSLSetProtocolVersionMin.restype = OSStatus + Security.SSLSetProtocolVersionMax.argtypes = [SSLContextRef, SSLProtocol] + Security.SSLSetProtocolVersionMax.restype = OSStatus + Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p] + Security.SecCopyErrorMessageString.restype = CFStringRef + Security.SSLReadFunc = SSLReadFunc + Security.SSLWriteFunc = SSLWriteFunc + Security.SSLContextRef = SSLContextRef + Security.SSLProtocol = SSLProtocol + Security.SSLCipherSuite = SSLCipherSuite + Security.SecIdentityRef = SecIdentityRef + Security.SecKeychainRef = SecKeychainRef + Security.SecTrustRef = SecTrustRef + Security.SecTrustResultType = SecTrustResultType + Security.SecExternalFormat = SecExternalFormat + Security.OSStatus = OSStatus + Security.kSecImportExportPassphrase = CFStringRef.in_dll( + Security, 'kSecImportExportPassphrase' + ) + Security.kSecImportItemIdentity = CFStringRef.in_dll( + Security, 'kSecImportItemIdentity' + ) + # CoreFoundation time! + CoreFoundation.CFRetain.argtypes = [CFTypeRef] + CoreFoundation.CFRetain.restype = CFTypeRef + CoreFoundation.CFRelease.argtypes = [CFTypeRef] + CoreFoundation.CFRelease.restype = None + CoreFoundation.CFGetTypeID.argtypes = [CFTypeRef] + CoreFoundation.CFGetTypeID.restype = CFTypeID + CoreFoundation.CFStringCreateWithCString.argtypes = [ + CFAllocatorRef, c_char_p, CFStringEncoding + ] + CoreFoundation.CFStringCreateWithCString.restype = CFStringRef + CoreFoundation.CFStringGetCStringPtr.argtypes = [ + CFStringRef, CFStringEncoding + ] + CoreFoundation.CFStringGetCStringPtr.restype = c_char_p + CoreFoundation.CFStringGetCString.argtypes = [ + CFStringRef, c_char_p, CFIndex, CFStringEncoding + ] + CoreFoundation.CFStringGetCString.restype = c_bool + CoreFoundation.CFDataCreate.argtypes = [CFAllocatorRef, c_char_p, CFIndex] + CoreFoundation.CFDataCreate.restype = CFDataRef + CoreFoundation.CFDataGetLength.argtypes = [CFDataRef] + CoreFoundation.CFDataGetLength.restype = CFIndex + CoreFoundation.CFDataGetBytePtr.argtypes = [CFDataRef] + CoreFoundation.CFDataGetBytePtr.restype = c_void_p + CoreFoundation.CFDictionaryCreate.argtypes = [ + CFAllocatorRef, + POINTER(CFTypeRef), + POINTER(CFTypeRef), + CFIndex, + CFDictionaryKeyCallBacks, + CFDictionaryValueCallBacks, + ] + CoreFoundation.CFDictionaryCreate.restype = CFDictionaryRef + CoreFoundation.CFDictionaryGetValue.argtypes = [CFDictionaryRef, CFTypeRef] + CoreFoundation.CFDictionaryGetValue.restype = CFTypeRef + CoreFoundation.CFArrayCreate.argtypes = [ + CFAllocatorRef, POINTER(CFTypeRef), CFIndex, CFArrayCallBacks + ] + CoreFoundation.CFArrayCreate.restype = CFArrayRef + CoreFoundation.CFArrayCreateMutable.argtypes = [ + CFAllocatorRef, CFIndex, CFArrayCallBacks + ] + CoreFoundation.CFArrayCreateMutable.restype = CFMutableArrayRef + CoreFoundation.CFArrayAppendValue.argtypes = [CFMutableArrayRef, c_void_p] + CoreFoundation.CFArrayAppendValue.restype = None + CoreFoundation.CFArrayGetCount.argtypes = [CFArrayRef] + CoreFoundation.CFArrayGetCount.restype = CFIndex + CoreFoundation.CFArrayGetValueAtIndex.argtypes = [CFArrayRef, CFIndex] + CoreFoundation.CFArrayGetValueAtIndex.restype = c_void_p + CoreFoundation.kCFAllocatorDefault = CFAllocatorRef.in_dll( + CoreFoundation, 'kCFAllocatorDefault' + ) + CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll( + CoreFoundation, 'kCFTypeArrayCallBacks' + ) + CoreFoundation.kCFTypeDictionaryKeyCallBacks = c_void_p.in_dll( + CoreFoundation, 'kCFTypeDictionaryKeyCallBacks' + ) + CoreFoundation.kCFTypeDictionaryValueCallBacks = c_void_p.in_dll( + CoreFoundation, 'kCFTypeDictionaryValueCallBacks' + ) + CoreFoundation.CFTypeRef = CFTypeRef + CoreFoundation.CFArrayRef = CFArrayRef + CoreFoundation.CFStringRef = CFStringRef + CoreFoundation.CFDictionaryRef = CFDictionaryRef +except (AttributeError): + raise ImportError('Error initializing ctypes') + + +class CFConst(object): + """ + A class object that acts as essentially a namespace for CoreFoundation + constants. + """ + kCFStringEncodingUTF8 = CFStringEncoding(0x08000100) + + +class SecurityConst(object): + """ + A class object that acts as essentially a namespace for Security constants. + """ + kSSLSessionOptionBreakOnServerAuth = 0 + kSSLProtocol2 = 1 + kSSLProtocol3 = 2 + kTLSProtocol1 = 4 + kTLSProtocol11 = 7 + kTLSProtocol12 = 8 + kSSLClientSide = 1 + kSSLStreamType = 0 + kSecFormatPEMSequence = 10 + kSecTrustResultInvalid = 0 + kSecTrustResultProceed = 1 + # This gap is present on purpose: this was kSecTrustResultConfirm, which + # is deprecated. + kSecTrustResultDeny = 3 + kSecTrustResultUnspecified = 4 + kSecTrustResultRecoverableTrustFailure = 5 + kSecTrustResultFatalTrustFailure = 6 + kSecTrustResultOtherError = 7 + errSSLProtocol = -9800 + errSSLWouldBlock = -9803 + errSSLClosedGraceful = -9805 + errSSLClosedNoNotify = -9816 + errSSLClosedAbort = -9806 + errSSLXCertChainInvalid = -9807 + errSSLCrypto = -9809 + errSSLInternal = -9810 + errSSLCertExpired = -9814 + errSSLCertNotYetValid = -9815 + errSSLUnknownRootCert = -9812 + errSSLNoRootCert = -9813 + errSSLHostNameMismatch = -9843 + errSSLPeerHandshakeFail = -9824 + errSSLPeerUserCancelled = -9839 + errSSLWeakPeerEphemeralDHKey = -9850 + errSSLServerAuthCompleted = -9841 + errSSLRecordOverflow = -9847 + errSecVerifyFailed = -67808 + errSecNoTrustSettings = -25263 + errSecItemNotFound = -25300 + errSecInvalidTrustSettings = -25262 + # Cipher suites. We only pick the ones our default cipher string allows. + TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02C + TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 = 0xC030 + TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02B + TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 = 0xC02F + TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 = 0x00A3 + TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 = 0x009F + TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 = 0x00A2 + TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 = 0x009E + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC024 + TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xC028 + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA = 0xC00A + TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA = 0xC014 + TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 = 0x006B + TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 = 0x006A + TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x0039 + TLS_DHE_DSS_WITH_AES_256_CBC_SHA = 0x0038 + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC023 + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 = 0xC027 + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA = 0xC009 + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA = 0xC013 + TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 = 0x0067 + TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 = 0x0040 + TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x0033 + TLS_DHE_DSS_WITH_AES_128_CBC_SHA = 0x0032 + TLS_RSA_WITH_AES_256_GCM_SHA384 = 0x009D + TLS_RSA_WITH_AES_128_GCM_SHA256 = 0x009C + TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x003D + TLS_RSA_WITH_AES_128_CBC_SHA256 = 0x003C + TLS_RSA_WITH_AES_256_CBC_SHA = 0x0035 + TLS_RSA_WITH_AES_128_CBC_SHA = 0x002F + TLS_AES_128_GCM_SHA256 = 0x1301 + TLS_AES_256_GCM_SHA384 = 0x1302 + TLS_CHACHA20_POLY1305_SHA256 = 0x1303 diff --git a/requests3/core/_http/contrib/_securetransport/low_level.py b/requests3/core/_http/contrib/_securetransport/low_level.py new file mode 100644 index 00000000..3c7cee3e --- /dev/null +++ b/requests3/core/_http/contrib/_securetransport/low_level.py @@ -0,0 +1,313 @@ +""" +Low-level helpers for the SecureTransport bindings. + +These are Python functions that are not directly related to the high-level APIs +but are necessary to get them to work. They include a whole bunch of low-level +CoreFoundation messing about and memory management. The concerns in this module +are almost entirely about trying to avoid memory leaks and providing +appropriate and useful assistance to the higher-level code. +""" +import base64 +import ctypes +import itertools +import re +import os +import ssl +import tempfile + +from .bindings import Security, CoreFoundation, CFConst + +# This regular expression is used to grab PEM data out of a PEM bundle. +_PEM_CERTS_RE = re.compile( + b"-----BEGIN CERTIFICATE-----\n(.*?)\n-----END CERTIFICATE-----", re.DOTALL +) + + +def _cf_data_from_bytes(bytestring): + """ + Given a bytestring, create a CFData object from it. This CFData object must + be CFReleased by the caller. + """ + return CoreFoundation.CFDataCreate( + CoreFoundation.kCFAllocatorDefault, bytestring, len(bytestring) + ) + + +def _cf_dictionary_from_tuples(tuples): + """ + Given a list of Python tuples, create an associated CFDictionary. + """ + dictionary_size = len(tuples) + # We need to get the dictionary keys and values out in the same order. + keys = (t[0] for t in tuples) + values = (t[1] for t in tuples) + cf_keys = (CoreFoundation.CFTypeRef * dictionary_size)(*keys) + cf_values = (CoreFoundation.CFTypeRef * dictionary_size)(*values) + return CoreFoundation.CFDictionaryCreate( + CoreFoundation.kCFAllocatorDefault, + cf_keys, + cf_values, + dictionary_size, + CoreFoundation.kCFTypeDictionaryKeyCallBacks, + CoreFoundation.kCFTypeDictionaryValueCallBacks, + ) + + +def _cf_string_to_unicode(value): + """ + Creates a Unicode string from a CFString object. Used entirely for error + reporting. + + Yes, it annoys me quite a lot that this function is this complex. + """ + value_as_void_p = ctypes.cast(value, ctypes.POINTER(ctypes.c_void_p)) + string = CoreFoundation.CFStringGetCStringPtr( + value_as_void_p, CFConst.kCFStringEncodingUTF8 + ) + if string is None: + buffer = ctypes.create_string_buffer(1024) + result = CoreFoundation.CFStringGetCString( + value_as_void_p, buffer, 1024, CFConst.kCFStringEncodingUTF8 + ) + if not result: + raise OSError('Error copying C string from CFStringRef') + + string = buffer.value + if string is not None: + string = string.decode('utf-8') + return string + + +def _assert_no_error(error, exception_class=None): + """ + Checks the return code and throws an exception if there is an error to + report + """ + if error == 0: + return + + cf_error_string = Security.SecCopyErrorMessageString(error, None) + output = _cf_string_to_unicode(cf_error_string) + CoreFoundation.CFRelease(cf_error_string) + if output is None or output == u'': + output = u'OSStatus %s' % error + if exception_class is None: + exception_class = ssl.SSLError + raise exception_class(output) + + +def _cert_array_from_pem(pem_bundle): + """ + Given a bundle of certs in PEM format, turns them into a CFArray of certs + that can be used to validate a cert chain. + """ + der_certs = [ + base64.b64decode(match.group(1)) + for match in _PEM_CERTS_RE.finditer(pem_bundle) + ] + if not der_certs: + raise ssl.SSLError("No root certificates specified") + + cert_array = CoreFoundation.CFArrayCreateMutable( + CoreFoundation.kCFAllocatorDefault, + 0, + ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), + ) + if not cert_array: + raise ssl.SSLError("Unable to allocate memory!") + + try: + for der_bytes in der_certs: + certdata = _cf_data_from_bytes(der_bytes) + if not certdata: + raise ssl.SSLError("Unable to allocate memory!") + + cert = Security.SecCertificateCreateWithData( + CoreFoundation.kCFAllocatorDefault, certdata + ) + CoreFoundation.CFRelease(certdata) + if not cert: + raise ssl.SSLError("Unable to build cert object!") + + CoreFoundation.CFArrayAppendValue(cert_array, cert) + CoreFoundation.CFRelease(cert) + except Exception: + # We need to free the array before the exception bubbles further. + # We only want to do that if an error occurs: otherwise, the caller + # should free. + CoreFoundation.CFRelease(cert_array) + return cert_array + + +def _is_cert(item): + """ + Returns True if a given CFTypeRef is a certificate. + """ + expected = Security.SecCertificateGetTypeID() + return CoreFoundation.CFGetTypeID(item) == expected + + +def _is_identity(item): + """ + Returns True if a given CFTypeRef is an identity. + """ + expected = Security.SecIdentityGetTypeID() + return CoreFoundation.CFGetTypeID(item) == expected + + +def _temporary_keychain(): + """ + This function creates a temporary Mac keychain that we can use to work with + credentials. This keychain uses a one-time password and a temporary file to + store the data. We expect to have one keychain per socket. The returned + SecKeychainRef must be freed by the caller, including calling + SecKeychainDelete. + + Returns a tuple of the SecKeychainRef and the path to the temporary + directory that contains it. + """ + # Unfortunately, SecKeychainCreate requires a path to a keychain. This + # means we cannot use mkstemp to use a generic temporary file. Instead, + # we're going to create a temporary directory and a filename to use there. + # This filename will be 8 random bytes expanded into base64. We also need + # some random bytes to password-protect the keychain we're creating, so we + # ask for 40 random bytes. + random_bytes = os.urandom(40) + filename = base64.b64encode(random_bytes[:8]).decode('utf-8') + password = base64.b64encode(random_bytes[8:]) # Must be valid UTF-8 + tempdirectory = tempfile.mkdtemp() + keychain_path = os.path.join(tempdirectory, filename).encode('utf-8') + # We now want to create the keychain itself. + keychain = Security.SecKeychainRef() + status = Security.SecKeychainCreate( + keychain_path, + len(password), + password, + False, + None, + ctypes.byref(keychain), + ) + _assert_no_error(status) + # Having created the keychain, we want to pass it off to the caller. + return keychain, tempdirectory + + +def _load_items_from_file(keychain, path): + """ + Given a single file, loads all the trust objects from it into arrays and + the keychain. + Returns a tuple of lists: the first list is a list of identities, the + second a list of certs. + """ + certificates = [] + identities = [] + result_array = None + with open(path, 'rb') as f: + raw_filedata = f.read() + try: + filedata = CoreFoundation.CFDataCreate( + CoreFoundation.kCFAllocatorDefault, raw_filedata, len(raw_filedata) + ) + result_array = CoreFoundation.CFArrayRef() + result = Security.SecItemImport( + filedata, # cert data + None, # Filename, leaving it out for now + None, # What the type of the file is, we don't care + None, # what's in the file, we don't care + 0, # import flags + None, # key params, can include passphrase in the future + keychain, # The keychain to insert into + ctypes.byref(result_array), # Results + ) + _assert_no_error(result) + # A CFArray is not very useful to us as an intermediary + # representation, so we are going to extract the objects we want + # and then free the array. We don't need to keep hold of keys: the + # keychain already has them! + result_count = CoreFoundation.CFArrayGetCount(result_array) + for index in range(result_count): + item = CoreFoundation.CFArrayGetValueAtIndex(result_array, index) + item = ctypes.cast(item, CoreFoundation.CFTypeRef) + if _is_cert(item): + CoreFoundation.CFRetain(item) + certificates.append(item) + elif _is_identity(item): + CoreFoundation.CFRetain(item) + identities.append(item) + finally: + if result_array: + CoreFoundation.CFRelease(result_array) + CoreFoundation.CFRelease(filedata) + return (identities, certificates) + + +def _load_client_cert_chain(keychain, *paths): + """ + Load certificates and maybe keys from a number of files. Has the end goal + of returning a CFArray containing one SecIdentityRef, and then zero or more + SecCertificateRef objects, suitable for use as a client certificate trust + chain. + """ + # Ok, the strategy. + # + # This relies on knowing that macOS will not give you a SecIdentityRef + # unless you have imported a key into a keychain. This is a somewhat + # artificial limitation of macOS (for example, it doesn't necessarily + # affect iOS), but there is nothing inside Security.framework that lets you + # get a SecIdentityRef without having a key in a keychain. + # + # So the policy here is we take all the files and iterate them in order. + # Each one will use SecItemImport to have one or more objects loaded from + # it. We will also point at a keychain that macOS can use to work with the + # private key. + # + # Once we have all the objects, we'll check what we actually have. If we + # already have a SecIdentityRef in hand, fab: we'll use that. Otherwise, + # we'll take the first certificate (which we assume to be our leaf) and + # ask the keychain to give us a SecIdentityRef with that cert's associated + # key. + # + # We'll then return a CFArray containing the trust chain: one + # SecIdentityRef and then zero-or-more SecCertificateRef objects. The + # responsibility for freeing this CFArray will be with the caller. This + # CFArray must remain alive for the entire connection, so in practice it + # will be stored with a single SSLSocket, along with the reference to the + # keychain. + certificates = [] + identities = [] + # Filter out bad paths. + paths = (path for path in paths if path) + try: + for file_path in paths: + new_identities, new_certs = _load_items_from_file( + keychain, file_path + ) + identities.extend(new_identities) + certificates.extend(new_certs) + # Ok, we have everything. The question is: do we have an identity? If + # not, we want to grab one from the first cert we have. + if not identities: + new_identity = Security.SecIdentityRef() + status = Security.SecIdentityCreateWithCertificate( + keychain, certificates[0], ctypes.byref(new_identity) + ) + _assert_no_error(status) + identities.append(new_identity) + # We now want to release the original certificate, as we no longer + # need it. + CoreFoundation.CFRelease(certificates.pop(0)) + # We now need to build a new CFArray that holds the trust chain. + trust_chain = CoreFoundation.CFArrayCreateMutable( + CoreFoundation.kCFAllocatorDefault, + 0, + ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), + ) + for item in itertools.chain(identities, certificates): + # ArrayAppendValue does a CFRetain on the item. That's fine, + # because the finally block will release our other refs to them. + CoreFoundation.CFArrayAppendValue(trust_chain, item) + return trust_chain + + finally: + for obj in itertools.chain(identities, certificates): + CoreFoundation.CFRelease(obj) diff --git a/requests3/core/_http/contrib/appengine.py b/requests3/core/_http/contrib/appengine.py new file mode 100644 index 00000000..62d58fb6 --- /dev/null +++ b/requests3/core/_http/contrib/appengine.py @@ -0,0 +1,332 @@ +""" +This module provides a pool manager that uses Google App Engine's +`URLFetch Service `_. + +Example usage:: + + from urllib3 import PoolManager + from urllib3.contrib.appengine import AppEngineManager, is_appengine_sandbox + + if is_appengine_sandbox(): + # AppEngineManager uses AppEngine's URLFetch API behind the scenes + http = AppEngineManager() + else: + # PoolManager uses a socket-level API behind the scenes + http = PoolManager() + + r = http.request('GET', 'https://google.com/') + +There are `limitations `_ to the URLFetch service and it may not be +the best choice for your application. There are three options for using +urllib3 on Google App Engine: + +1. You can use :class:`AppEngineManager` with URLFetch. URLFetch is + cost-effective in many circumstances as long as your usage is within the + limitations. +2. You can use a normal :class:`~urllib3.PoolManager` by enabling sockets. + Sockets also have `limitations and restrictions + `_ and have a lower free quota than URLFetch. + To use sockets, be sure to specify the following in your ``app.yaml``:: + + env_variables: + GAE_USE_SOCKETS_HTTPLIB : 'true' + +3. If you are using `App Engine Flexible +`_, you can use the standard +:class:`PoolManager` without any configuration or special environment variables. +""" + +from __future__ import absolute_import +import logging +import os +import warnings +from ..packages.six.moves.urllib.parse import urljoin + +from ..exceptions import ( + HTTPError, + HTTPWarning, + MaxRetryError, + ProtocolError, + TimeoutError, + SSLError, +) + +from ..packages.six import BytesIO +from ..request import RequestMethods +from ..response import HTTPResponse +from ..util.timeout import Timeout +from ..util.retry import Retry + +try: + from google.appengine.api import urlfetch +except ImportError: + urlfetch = None +log = logging.getLogger(__name__) + + +class AppEnginePlatformWarning(HTTPWarning): + pass + + +class AppEnginePlatformError(HTTPError): + pass + + +class AppEngineManager(RequestMethods): + """ + Connection manager for Google App Engine sandbox applications. + + This manager uses the URLFetch service directly instead of using the + emulated httplib, and is subject to URLFetch limitations as described in + the App Engine documentation `here + `_. + + Notably it will raise an :class:`AppEnginePlatformError` if: + * URLFetch is not available. + * If you attempt to use this on App Engine Flexible, as full socket + support is available. + * If a request size is more than 10 megabytes. + * If a response size is more than 32 megabtyes. + * If you use an unsupported request method such as OPTIONS. + + Beyond those cases, it will raise normal urllib3 errors. + """ + + def __init__( + self, + headers=None, + retries=None, + validate_certificate=True, + urlfetch_retries=True, + ): + if not urlfetch: + raise AppEnginePlatformError( + "URLFetch is not available in this environment." + ) + + if is_prod_appengine_mvms(): + raise AppEnginePlatformError( + "Use normal urllib3.PoolManager instead of AppEngineManager" + "on Managed VMs, as using URLFetch is not necessary in " + "this environment." + ) + + warnings.warn( + "urllib3 is using URLFetch on Google App Engine sandbox instead " + "of sockets. To use sockets directly instead of URLFetch see " + "https://urllib3.readthedocs.io/en/latest/reference/urllib3.contrib.html.", + AppEnginePlatformWarning, + ) + RequestMethods.__init__(self, headers) + self.validate_certificate = validate_certificate + self.urlfetch_retries = urlfetch_retries + self.retries = retries or Retry.DEFAULT + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Return False to re-raise any potential exceptions + return False + + def urlopen( + self, + method, + url, + body=None, + headers=None, + retries=None, + redirect=True, + timeout=Timeout.DEFAULT_TIMEOUT, + **response_kw + ): + retries = self._get_retries(retries, redirect) + try: + follow_redirects = ( + redirect and retries.redirect != 0 and retries.total + ) + response = urlfetch.fetch( + url, + payload=body, + method=method, + headers=headers or {}, + allow_truncated=False, + follow_redirects=self.urlfetch_retries and follow_redirects, + deadline=self._get_absolute_timeout(timeout), + validate_certificate=self.validate_certificate, + ) + except urlfetch.DeadlineExceededError as e: + raise TimeoutError(self, e) + + except urlfetch.InvalidURLError as e: + if 'too large' in str(e): + raise AppEnginePlatformError( + "URLFetch request too large, URLFetch only " + "supports requests up to 10mb in size.", + e, + ) + + raise ProtocolError(e) + + except urlfetch.DownloadError as e: + if 'Too many redirects' in str(e): + raise MaxRetryError(self, url, reason=e) + + raise ProtocolError(e) + + except urlfetch.ResponseTooLargeError as e: + raise AppEnginePlatformError( + "URLFetch response too large, URLFetch only supports" + "responses up to 32mb in size.", + e, + ) + + except urlfetch.SSLCertificateError as e: + raise SSLError(e) + + except urlfetch.InvalidMethodError as e: + raise AppEnginePlatformError( + "URLFetch does not support method: %s" % method, e + ) + + http_response = self._urlfetch_response_to_http_response( + response, retries=retries, **response_kw + ) + # Handle redirect? + redirect_location = redirect and http_response.get_redirect_location() + if redirect_location: + # Check for redirect response + if (self.urlfetch_retries and retries.raise_on_redirect): + raise MaxRetryError(self, url, "too many redirects") + + else: + if http_response.status == 303: + method = 'GET' + try: + retries = retries.increment( + method, url, response=http_response, _pool=self + ) + except MaxRetryError: + if retries.raise_on_redirect: + raise MaxRetryError(self, url, "too many redirects") + + return http_response + + retries.sleep_for_retry(http_response) + log.debug("Redirecting %s -> %s", url, redirect_location) + redirect_url = urljoin(url, redirect_location) + return self.urlopen( + method, + redirect_url, + body, + headers, + retries=retries, + redirect=redirect, + timeout=timeout, + **response_kw + ) + + # Check if we should retry the HTTP response. + has_retry_after = bool(http_response.getheader('Retry-After')) + if retries.is_retry(method, http_response.status, has_retry_after): + retries = retries.increment( + method, url, response=http_response, _pool=self + ) + log.debug("Retry: %s", url) + retries.sleep(http_response) + return self.urlopen( + method, + url, + body=body, + headers=headers, + retries=retries, + redirect=redirect, + timeout=timeout, + **response_kw + ) + + return http_response + + def _urlfetch_response_to_http_response( + self, urlfetch_resp, **response_kw + ): + if is_prod_appengine(): + # Production GAE handles deflate encoding automatically, but does + # not remove the encoding header. + content_encoding = urlfetch_resp.headers.get('content-encoding') + if content_encoding == 'deflate': + del urlfetch_resp.headers['content-encoding'] + transfer_encoding = urlfetch_resp.headers.get('transfer-encoding') + # We have a full response's content, + # so let's make sure we don't report ourselves as chunked data. + if transfer_encoding == 'chunked': + encodings = transfer_encoding.split(",") + encodings.remove('chunked') + urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings) + return HTTPResponse( + # In order for decoding to work, we must present the content as + # a file-like object. + body=BytesIO(urlfetch_resp.content), + headers=urlfetch_resp.headers, + status=urlfetch_resp.status_code, + **response_kw + ) + + def _get_absolute_timeout(self, timeout): + if timeout is Timeout.DEFAULT_TIMEOUT: + return None # Defer to URLFetch's default. + + if isinstance(timeout, Timeout): + if timeout._read is not None or timeout._connect is not None: + warnings.warn( + "URLFetch does not support granular timeout settings, " + "reverting to total or default URLFetch timeout.", + AppEnginePlatformWarning, + ) + return timeout.total + + return timeout + + def _get_retries(self, retries, redirect): + if not isinstance(retries, Retry): + retries = Retry.from_int( + retries, redirect=redirect, default=self.retries + ) + if retries.connect or retries.read or retries.redirect: + warnings.warn( + "URLFetch only supports total retries and does not " + "recognize connect, read, or redirect retry parameters.", + AppEnginePlatformWarning, + ) + return retries + + +def is_appengine(): + return ( + is_local_appengine() or is_prod_appengine() or is_prod_appengine_mvms() + ) + + +def is_appengine_sandbox(): + return is_appengine() and not is_prod_appengine_mvms() + + +def is_local_appengine(): + return ( + 'APPENGINE_RUNTIME' in os.environ and + 'Development/' in os.environ['SERVER_SOFTWARE'] + ) + + +def is_prod_appengine(): + return ( + 'APPENGINE_RUNTIME' in os.environ and + 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and + not is_prod_appengine_mvms() + ) + + +def is_prod_appengine_mvms(): + return os.environ.get('GAE_VM', False) == 'true' diff --git a/requests3/core/_http/contrib/pyopenssl.py b/requests3/core/_http/contrib/pyopenssl.py new file mode 100644 index 00000000..c7884b0c --- /dev/null +++ b/requests3/core/_http/contrib/pyopenssl.py @@ -0,0 +1,485 @@ +""" +SSL with SNI_-support for Python 2. Follow these instructions if you would +like to verify SSL certificates in Python 2. Note, the default libraries do +*not* do certificate checking; you need to do additional work to validate +certificates yourself. + +This needs the following packages installed: + +* pyOpenSSL (tested with 16.0.0) +* cryptography (minimum 1.3.4, from pyopenssl) +* idna (minimum 2.0, from cryptography) + +However, pyopenssl depends on cryptography, which depends on idna, so while we +use all three directly here we end up having relatively few packages required. + +You can install them with the following command: + + pip install pyopenssl cryptography idna + +To activate certificate checking, call +:func:`~urllib3.contrib.pyopenssl.inject_into_urllib3` from your Python code +before you begin making HTTP requests. This can be done in a ``sitecustomize`` +module, or at any other time before your application begins using ``urllib3``, +like this:: + + try: + import urllib3.contrib.pyopenssl + urllib3.contrib.pyopenssl.inject_into_urllib3() + except ImportError: + pass + +Now you can use :mod:`urllib3` as you normally would, and it will support SNI +when the required modules are installed. + +Activating this module also has the positive side effect of disabling SSL/TLS +compression in Python 2 (see `CRIME attack`_). + +If you want to configure the default list of supported cipher suites, you can +set the ``urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST`` variable. + +.. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication +.. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit) +""" +from __future__ import absolute_import + +import OpenSSL.SSL +from cryptography import x509 +from cryptography.hazmat.backends.openssl import backend as openssl_backend +from cryptography.hazmat.backends.openssl.x509 import _Certificate + +from socket import timeout, error as SocketError +from io import BytesIO + +try: # Platform-specific: Python 2 + from socket import _fileobject +except ImportError: # Platform-specific: Python 3 + _fileobject = None + from ..packages.backports.makefile import backport_makefile +import logging +import ssl +from ..packages import six +import sys + +from .. import util + +__all__ = ['inject_into_urllib3', 'extract_from_urllib3'] +# SNI always works. +HAS_SNI = True +# Map from urllib3 to PyOpenSSL compatible parameter-values. +_openssl_versions = { + ssl.PROTOCOL_SSLv23: OpenSSL.SSL.SSLv23_METHOD, + ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, +} +if hasattr(ssl, 'PROTOCOL_TLSv1_1') and hasattr(OpenSSL.SSL, 'TLSv1_1_METHOD'): + _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD +if hasattr(ssl, 'PROTOCOL_TLSv1_2') and hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'): + _openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD +try: + _openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD}) +except AttributeError: + pass +_stdlib_to_openssl_verify = { + ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE, + ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER, + ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER + + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, +} +_openssl_to_stdlib_verify = dict( + (v, k) for k, v in _stdlib_to_openssl_verify.items() +) +# OpenSSL will only write 16K at a time +SSL_WRITE_BLOCKSIZE = 16384 +orig_util_HAS_SNI = util.HAS_SNI +orig_util_SSLContext = util.ssl_.SSLContext +log = logging.getLogger(__name__) + + +def inject_into_urllib3(): + 'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.' + _validate_dependencies_met() + util.ssl_.SSLContext = PyOpenSSLContext + util.HAS_SNI = HAS_SNI + util.ssl_.HAS_SNI = HAS_SNI + util.IS_PYOPENSSL = True + util.ssl_.IS_PYOPENSSL = True + + +def extract_from_urllib3(): + 'Undo monkey-patching by :func:`inject_into_urllib3`.' + util.ssl_.SSLContext = orig_util_SSLContext + util.HAS_SNI = orig_util_HAS_SNI + util.ssl_.HAS_SNI = orig_util_HAS_SNI + util.IS_PYOPENSSL = False + util.ssl_.IS_PYOPENSSL = False + + +def _validate_dependencies_met(): + """ + Verifies that PyOpenSSL's package-level dependencies have been met. + Throws `ImportError` if they are not met. + """ + # Method added in `cryptography==1.1`; not available in older versions + from cryptography.x509.extensions import Extensions + + if getattr(Extensions, "get_extension_for_class", None) is None: + raise ImportError( + "'cryptography' module missing required functionality. " + "Try upgrading to v1.3.4 or newer." + ) + + # pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509 + # attribute is only present on those versions. + from OpenSSL.crypto import X509 + + x509 = X509() + if getattr(x509, "_x509", None) is None: + raise ImportError( + "'pyOpenSSL' module missing required functionality. " + "Try upgrading to v0.14 or newer." + ) + + +def _dnsname_to_stdlib(name): + """ + Converts a dNSName SubjectAlternativeName field to the form used by the + standard library on the given Python version. + + Cryptography produces a dNSName as a unicode string that was idna-decoded + from ASCII bytes. We need to idna-encode that string to get it back, and + then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib + uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8). + """ + + def idna_encode(name): + """ + Borrowed wholesale from the Python Cryptography Project. It turns out + that we can't just safely call `idna.encode`: it can explode for + wildcard names. This avoids that problem. + """ + import idna + + for prefix in [u'*.', u'.']: + if name.startswith(prefix): + name = name[len(prefix):] + return prefix.encode('ascii') + idna.encode(name) + + return idna.encode(name) + + name = idna_encode(name) + if sys.version_info >= (3, 0): + name = name.decode('utf-8') + return name + + +def get_subj_alt_name(peer_cert): + """ + Given an PyOpenSSL certificate, provides all the subject alternative names. + """ + # Pass the cert to cryptography, which has much better APIs for this. + if hasattr(peer_cert, "to_cryptography"): + cert = peer_cert.to_cryptography() + else: + # This is technically using private APIs, but should work across all + # relevant versions before PyOpenSSL got a proper API for this. + cert = _Certificate(openssl_backend, peer_cert._x509) + # We want to find the SAN extension. Ask Cryptography to locate it (it's + # faster than looping in Python) + try: + ext = cert.extensions.get_extension_for_class( + x509.SubjectAlternativeName + ).value + except x509.ExtensionNotFound: + # No such extension, return the empty list. + return [] + + except ( + x509.DuplicateExtension, + x509.UnsupportedExtension, + x509.UnsupportedGeneralNameType, + UnicodeError, + ) as e: + # A problem has been found with the quality of the certificate. Assume + # no SAN field is present. + log.warning( + "A problem was encountered with the certificate that prevented " + "urllib3 from finding the SubjectAlternativeName field. This can " + "affect certificate validation. The error was %s", + e, + ) + return [] + + # We want to return dNSName and iPAddress fields. We need to cast the IPs + # back to strings because the match_hostname function wants them as + # strings. + # Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8 + # decoded. This is pretty frustrating, but that's what the standard library + # does with certificates, and so we need to attempt to do the same. + names = [ + ('DNS', _dnsname_to_stdlib(name)) + for name in ext.get_values_for_type(x509.DNSName) + ] + names.extend( + ('IP Address', str(name)) + for name in ext.get_values_for_type(x509.IPAddress) + ) + return names + + +class WrappedSocket(object): + '''API-compatibility wrapper for Python OpenSSL's Connection-class. + + Note: _makefile_refs, _drop() and _reuse() are needed for the garbage + collector of pypy. + ''' + + def __init__(self, connection, socket, suppress_ragged_eofs=True): + self.connection = connection + self.socket = socket + self.suppress_ragged_eofs = suppress_ragged_eofs + self._makefile_refs = 0 + self._closed = False + + def fileno(self): + return self.socket.fileno() + + + # Copy-pasted from Python 3.5 source code + def _decref_socketios(self): + if self._makefile_refs > 0: + self._makefile_refs -= 1 + if self._closed: + self.close() + + def recv(self, *args, **kwargs): + try: + data = self.connection.recv(*args, **kwargs) + except OpenSSL.SSL.SysCallError as e: + if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): + return b'' + + else: + raise SocketError(str(e)) + + except OpenSSL.SSL.ZeroReturnError as e: + if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: + return b'' + + else: + raise + + except OpenSSL.SSL.WantReadError: + rd = util.wait_for_read(self.socket, self.socket.gettimeout()) + if not rd: + raise timeout('The read operation timed out') + + else: + return self.recv(*args, **kwargs) + + else: + return data + + def recv_into(self, *args, **kwargs): + try: + return self.connection.recv_into(*args, **kwargs) + + except OpenSSL.SSL.SysCallError as e: + if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): + return 0 + + else: + raise SocketError(str(e)) + + except OpenSSL.SSL.ZeroReturnError as e: + if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: + return 0 + + else: + raise + + except OpenSSL.SSL.WantReadError: + rd = util.wait_for_read(self.socket, self.socket.gettimeout()) + if not rd: + raise timeout('The read operation timed out') + + else: + return self.recv_into(*args, **kwargs) + + def settimeout(self, timeout): + return self.socket.settimeout(timeout) + + def _send_until_done(self, data): + while True: + try: + return self.connection.send(data) + + except OpenSSL.SSL.WantWriteError: + wr = util.wait_for_write(self.socket, self.socket.gettimeout()) + if not wr: + raise timeout() + + continue + + except OpenSSL.SSL.SysCallError as e: + raise SocketError(str(e)) + + def send(self, data): + return self._send_until_done(data) + + def sendall(self, data): + total_sent = 0 + while total_sent < len(data): + sent = self._send_until_done( + data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE] + ) + total_sent += sent + + def shutdown(self): + # FIXME rethrow compatible exceptions should we ever use this + self.connection.shutdown() + + def close(self): + if self._makefile_refs < 1: + try: + self._closed = True + return self.connection.close() + + except OpenSSL.SSL.Error: + return + + else: + self._makefile_refs -= 1 + + def getpeercert(self, binary_form=False): + x509 = self.connection.get_peer_certificate() + if not x509: + return x509 + + if binary_form: + return OpenSSL.crypto.dump_certificate( + OpenSSL.crypto.FILETYPE_ASN1, x509 + ) + + return { + 'subject': ((('commonName', x509.get_subject().CN),),), + 'subjectAltName': get_subj_alt_name(x509), + } + + def setblocking(self, flag): + return self.connection.setblocking(flag) + + def _reuse(self): + self._makefile_refs += 1 + + def _drop(self): + if self._makefile_refs < 1: + self.close() + else: + self._makefile_refs -= 1 + + +if _fileobject: # Platform-specific: Python 2 + + def makefile(self, mode, bufsize=-1): + self._makefile_refs += 1 + return _fileobject(self, mode, bufsize, close=True) + + +else: # Platform-specific: Python 3 + makefile = backport_makefile +WrappedSocket.makefile = makefile + + +class PyOpenSSLContext(object): + """ + I am a wrapper class for the PyOpenSSL ``Context`` object. I am responsible + for translating the interface of the standard library ``SSLContext`` object + to calls into PyOpenSSL. + """ + + def __init__(self, protocol): + self.protocol = _openssl_versions[protocol] + self._ctx = OpenSSL.SSL.Context(self.protocol) + self._options = 0 + self.check_hostname = False + + @property + def options(self): + return self._options + + @options.setter + def options(self, value): + self._options = value + self._ctx.set_options(value) + + @property + def verify_mode(self): + return _openssl_to_stdlib_verify[self._ctx.get_verify_mode()] + + @verify_mode.setter + def verify_mode(self, value): + self._ctx.set_verify( + _stdlib_to_openssl_verify[value], _verify_callback + ) + + def set_default_verify_paths(self): + self._ctx.set_default_verify_paths() + + def set_ciphers(self, ciphers): + if isinstance(ciphers, six.text_type): + ciphers = ciphers.encode('utf-8') + self._ctx.set_cipher_list(ciphers) + + def load_verify_locations(self, cafile=None, capath=None, cadata=None): + if cafile is not None: + cafile = cafile.encode('utf-8') + if capath is not None: + capath = capath.encode('utf-8') + self._ctx.load_verify_locations(cafile, capath) + if cadata is not None: + self._ctx.load_verify_locations(BytesIO(cadata)) + + def load_cert_chain(self, certfile, keyfile=None, password=None): + self._ctx.use_certificate_chain_file(certfile) + if password is not None: + self._ctx.set_passwd_cb( + lambda max_length, prompt_twice, userdata: password + ) + self._ctx.use_privatekey_file(keyfile or certfile) + + def wrap_socket( + self, + sock, + server_side=False, + do_handshake_on_connect=True, + suppress_ragged_eofs=True, + server_hostname=None, + ): + cnx = OpenSSL.SSL.Connection(self._ctx, sock) + if isinstance( + server_hostname, six.text_type + ): # Platform-specific: Python 3 + server_hostname = server_hostname.encode('utf-8') + if server_hostname is not None: + cnx.set_tlsext_host_name(server_hostname) + cnx.set_connect_state() + while True: + try: + cnx.do_handshake() + except OpenSSL.SSL.WantReadError: + rd = util.wait_for_read(sock, sock.gettimeout()) + if not rd: + raise timeout('select timed out') + + continue + + except OpenSSL.SSL.Error as e: + raise ssl.SSLError('bad handshake: %r' % e) + + break + + return WrappedSocket(cnx, sock) + + +def _verify_callback(cnx, x509, err_no, err_depth, return_code): + return err_no == 0 diff --git a/requests3/core/_http/contrib/securetransport.py b/requests3/core/_http/contrib/securetransport.py new file mode 100644 index 00000000..4a92ad75 --- /dev/null +++ b/requests3/core/_http/contrib/securetransport.py @@ -0,0 +1,807 @@ +""" +SecureTranport support for urllib3 via ctypes. + +This makes platform-native TLS available to urllib3 users on macOS without the +use of a compiler. This is an important feature because the Python Package +Index is moving to become a TLSv1.2-or-higher server, and the default OpenSSL +that ships with macOS is not capable of doing TLSv1.2. The only way to resolve +this is to give macOS users an alternative solution to the problem, and that +solution is to use SecureTransport. + +We use ctypes here because this solution must not require a compiler. That's +because pip is not allowed to require a compiler either. + +This is not intended to be a seriously long-term solution to this problem. +The hope is that PEP 543 will eventually solve this issue for us, at which +point we can retire this contrib module. But in the short term, we need to +solve the impending tire fire that is Python on Mac without this kind of +contrib module. So...here we are. + +To use this module, simply import and inject it:: + + import urllib3.contrib.securetransport + urllib3.contrib.securetransport.inject_into_urllib3() + +Happy TLSing! +""" +from __future__ import absolute_import + +import contextlib +import ctypes +import errno +import os.path +import shutil +import socket +import ssl +import threading +import weakref + +from .. import util +from ._securetransport.bindings import ( + Security, SecurityConst, CoreFoundation +) +from ._securetransport.low_level import ( + _assert_no_error, + _cert_array_from_pem, + _temporary_keychain, + _load_client_cert_chain, +) + +try: # Platform-specific: Python 2 + from socket import _fileobject +except ImportError: # Platform-specific: Python 3 + _fileobject = None + from ..packages.backports.makefile import backport_makefile +try: + memoryview(b'') +except NameError: + raise ImportError("SecureTransport only works on Pythons with memoryview") + +__all__ = ['inject_into_urllib3', 'extract_from_urllib3'] +# SNI always works +HAS_SNI = True +orig_util_HAS_SNI = util.HAS_SNI +orig_util_SSLContext = util.ssl_.SSLContext +# This dictionary is used by the read callback to obtain a handle to the +# calling wrapped socket. This is a pretty silly approach, but for now it'll +# do. I feel like I should be able to smuggle a handle to the wrapped socket +# directly in the SSLConnectionRef, but for now this approach will work I +# guess. +# +# We need to lock around this structure for inserts, but we don't do it for +# reads/writes in the callbacks. The reasoning here goes as follows: +# +# 1. It is not possible to call into the callbacks before the dictionary is +# populated, so once in the callback the id must be in the dictionary. +# 2. The callbacks don't mutate the dictionary, they only read from it, and +# so cannot conflict with any of the insertions. +# +# This is good: if we had to lock in the callbacks we'd drastically slow down +# the performance of this code. +_connection_refs = weakref.WeakValueDictionary() +_connection_ref_lock = threading.Lock() +# Limit writes to 16kB. This is OpenSSL's limit, but we'll cargo-cult it over +# for no better reason than we need *a* limit, and this one is right there. +SSL_WRITE_BLOCKSIZE = 16384 +# This is our equivalent of util.ssl_.DEFAULT_CIPHERS, but expanded out to +# individual cipher suites. We need to do this becuase this is how +# SecureTransport wants them. +CIPHER_SUITES = [ + SecurityConst.TLS_AES_256_GCM_SHA384, + SecurityConst.TLS_CHACHA20_POLY1305_SHA256, + SecurityConst.TLS_AES_128_GCM_SHA256, + SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + SecurityConst.TLS_DHE_DSS_WITH_AES_256_GCM_SHA384, + SecurityConst.TLS_DHE_RSA_WITH_AES_256_GCM_SHA384, + SecurityConst.TLS_DHE_DSS_WITH_AES_128_GCM_SHA256, + SecurityConst.TLS_DHE_RSA_WITH_AES_128_GCM_SHA256, + SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, + SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, + SecurityConst.TLS_DHE_DSS_WITH_AES_256_CBC_SHA256, + SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA, + SecurityConst.TLS_DHE_DSS_WITH_AES_256_CBC_SHA, + SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, + SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, + SecurityConst.TLS_DHE_DSS_WITH_AES_128_CBC_SHA256, + SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA, + SecurityConst.TLS_DHE_DSS_WITH_AES_128_CBC_SHA, + SecurityConst.TLS_RSA_WITH_AES_256_GCM_SHA384, + SecurityConst.TLS_RSA_WITH_AES_128_GCM_SHA256, + SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA256, + SecurityConst.TLS_RSA_WITH_AES_128_CBC_SHA256, + SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA, + SecurityConst.TLS_RSA_WITH_AES_128_CBC_SHA, +] +# Basically this is simple: for PROTOCOL_SSLv23 we turn it into a low of +# TLSv1 and a high of TLSv1.2. For everything else, we pin to that version. +_protocol_to_min_max = { + ssl.PROTOCOL_SSLv23: ( + SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12 + ) +} +if hasattr(ssl, "PROTOCOL_SSLv2"): + _protocol_to_min_max[ssl.PROTOCOL_SSLv2] = ( + SecurityConst.kSSLProtocol2, SecurityConst.kSSLProtocol2 + ) +if hasattr(ssl, "PROTOCOL_SSLv3"): + _protocol_to_min_max[ssl.PROTOCOL_SSLv3] = ( + SecurityConst.kSSLProtocol3, SecurityConst.kSSLProtocol3 + ) +if hasattr(ssl, "PROTOCOL_TLSv1"): + _protocol_to_min_max[ssl.PROTOCOL_TLSv1] = ( + SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol1 + ) +if hasattr(ssl, "PROTOCOL_TLSv1_1"): + _protocol_to_min_max[ssl.PROTOCOL_TLSv1_1] = ( + SecurityConst.kTLSProtocol11, SecurityConst.kTLSProtocol11 + ) +if hasattr(ssl, "PROTOCOL_TLSv1_2"): + _protocol_to_min_max[ssl.PROTOCOL_TLSv1_2] = ( + SecurityConst.kTLSProtocol12, SecurityConst.kTLSProtocol12 + ) +if hasattr(ssl, "PROTOCOL_TLS"): + _protocol_to_min_max[ssl.PROTOCOL_TLS] = _protocol_to_min_max[ + ssl.PROTOCOL_SSLv23 + ] + + +def inject_into_urllib3(): + """ + Monkey-patch urllib3 with SecureTransport-backed SSL-support. + """ + util.ssl_.SSLContext = SecureTransportContext + util.HAS_SNI = HAS_SNI + util.ssl_.HAS_SNI = HAS_SNI + util.IS_SECURETRANSPORT = True + util.ssl_.IS_SECURETRANSPORT = True + + +def extract_from_urllib3(): + """ + Undo monkey-patching by :func:`inject_into_urllib3`. + """ + util.ssl_.SSLContext = orig_util_SSLContext + util.HAS_SNI = orig_util_HAS_SNI + util.ssl_.HAS_SNI = orig_util_HAS_SNI + util.IS_SECURETRANSPORT = False + util.ssl_.IS_SECURETRANSPORT = False + + +def _read_callback(connection_id, data_buffer, data_length_pointer): + """ + SecureTransport read callback. This is called by ST to request that data + be returned from the socket. + """ + wrapped_socket = None + try: + wrapped_socket = _connection_refs.get(connection_id) + if wrapped_socket is None: + return SecurityConst.errSSLInternal + + base_socket = wrapped_socket.socket + requested_length = data_length_pointer[0] + timeout = wrapped_socket.gettimeout() + error = None + read_count = 0 + buffer = (ctypes.c_char * requested_length).from_address(data_buffer) + buffer_view = memoryview(buffer) + try: + while read_count < requested_length: + if timeout is None or timeout >= 0: + readables = util.wait_for_read([base_socket], timeout) + if not readables: + raise socket.error(errno.EAGAIN, 'timed out') + + # We need to tell ctypes that we have a buffer that can be + # written to. Upsettingly, we do that like this: + chunk_size = base_socket.recv_into( + buffer_view[read_count:requested_length] + ) + read_count += chunk_size + if not chunk_size: + if not read_count: + return SecurityConst.errSSLClosedGraceful + + break + + except (socket.error) as e: + error = e.errno + if error is not None and error != errno.EAGAIN: + if error == errno.ECONNRESET: + return SecurityConst.errSSLClosedAbort + + raise + + data_length_pointer[0] = read_count + if read_count != requested_length: + return SecurityConst.errSSLWouldBlock + + return 0 + + except Exception as e: + if wrapped_socket is not None: + wrapped_socket._exception = e + return SecurityConst.errSSLInternal + + +def _write_callback(connection_id, data_buffer, data_length_pointer): + """ + SecureTransport write callback. This is called by ST to request that data + actually be sent on the network. + """ + wrapped_socket = None + try: + wrapped_socket = _connection_refs.get(connection_id) + if wrapped_socket is None: + return SecurityConst.errSSLInternal + + base_socket = wrapped_socket.socket + bytes_to_write = data_length_pointer[0] + data = ctypes.string_at(data_buffer, bytes_to_write) + timeout = wrapped_socket.gettimeout() + error = None + sent = 0 + try: + while sent < bytes_to_write: + if timeout is None or timeout >= 0: + writables = util.wait_for_write([base_socket], timeout) + if not writables: + raise socket.error(errno.EAGAIN, 'timed out') + + chunk_sent = base_socket.send(data) + sent += chunk_sent + # This has some needless copying here, but I'm not sure there's + # much value in optimising this data path. + data = data[chunk_sent:] + except (socket.error) as e: + error = e.errno + if error is not None and error != errno.EAGAIN: + if error == errno.ECONNRESET: + return SecurityConst.errSSLClosedAbort + + raise + + data_length_pointer[0] = sent + if sent != bytes_to_write: + return SecurityConst.errSSLWouldBlock + + return 0 + + except Exception as e: + if wrapped_socket is not None: + wrapped_socket._exception = e + return SecurityConst.errSSLInternal + + +# We need to keep these two objects references alive: if they get GC'd while +# in use then SecureTransport could attempt to call a function that is in freed +# memory. That would be...uh...bad. Yeah, that's the word. Bad. +_read_callback_pointer = Security.SSLReadFunc(_read_callback) +_write_callback_pointer = Security.SSLWriteFunc(_write_callback) + + +class WrappedSocket(object): + """ + API-compatibility wrapper for Python's OpenSSL wrapped socket object. + + Note: _makefile_refs, _drop(), and _reuse() are needed for the garbage + collector of PyPy. + """ + + def __init__(self, socket): + self.socket = socket + self.context = None + self._makefile_refs = 0 + self._closed = False + self._exception = None + self._keychain = None + self._keychain_dir = None + self._client_cert_chain = None + # We save off the previously-configured timeout and then set it to + # zero. This is done because we use select and friends to handle the + # timeouts, but if we leave the timeout set on the lower socket then + # Python will "kindly" call select on that socket again for us. Avoid + # that by forcing the timeout to zero. + self._timeout = self.socket.gettimeout() + self.socket.settimeout(0) + + @contextlib.contextmanager + def _raise_on_error(self): + """ + A context manager that can be used to wrap calls that do I/O from + SecureTransport. If any of the I/O callbacks hit an exception, this + context manager will correctly propagate the exception after the fact. + This avoids silently swallowing those exceptions. + + It also correctly forces the socket closed. + """ + self._exception = None + # We explicitly don't catch around this yield because in the unlikely + # event that an exception was hit in the block we don't want to swallow + # it. + yield + + if self._exception is not None: + exception, self._exception = self._exception, None + self.close() + raise exception + + def _set_ciphers(self): + """ + Sets up the allowed ciphers. By default this matches the set in + util.ssl_.DEFAULT_CIPHERS, at least as supported by macOS. This is done + custom and doesn't allow changing at this time, mostly because parsing + OpenSSL cipher strings is going to be a freaking nightmare. + """ + ciphers = (Security.SSLCipherSuite * len(CIPHER_SUITES))( + *CIPHER_SUITES + ) + result = Security.SSLSetEnabledCiphers( + self.context, ciphers, len(CIPHER_SUITES) + ) + _assert_no_error(result) + + def _custom_validate(self, verify, trust_bundle): + """ + Called when we have set custom validation. We do this in two cases: + first, when cert validation is entirely disabled; and second, when + using a custom trust DB. + """ + # If we disabled cert validation, just say: cool. + if not verify: + return + + # We want data in memory, so load it up. + if os.path.isfile(trust_bundle): + with open(trust_bundle, 'rb') as f: + trust_bundle = f.read() + cert_array = None + trust = Security.SecTrustRef() + try: + # Get a CFArray that contains the certs we want. + cert_array = _cert_array_from_pem(trust_bundle) + # Ok, now the hard part. We want to get the SecTrustRef that ST has + # created for this connection, shove our CAs into it, tell ST to + # ignore everything else it knows, and then ask if it can build a + # chain. This is a buuuunch of code. + result = Security.SSLCopyPeerTrust( + self.context, ctypes.byref(trust) + ) + _assert_no_error(result) + if not trust: + raise ssl.SSLError("Failed to copy trust reference") + + result = Security.SecTrustSetAnchorCertificates(trust, cert_array) + _assert_no_error(result) + result = Security.SecTrustSetAnchorCertificatesOnly(trust, True) + _assert_no_error(result) + trust_result = Security.SecTrustResultType() + result = Security.SecTrustEvaluate( + trust, ctypes.byref(trust_result) + ) + _assert_no_error(result) + finally: + if trust: + CoreFoundation.CFRelease(trust) + if cert_array is None: + CoreFoundation.CFRelease(cert_array) + # Ok, now we can look at what the result was. + successes = ( + SecurityConst.kSecTrustResultUnspecified, + SecurityConst.kSecTrustResultProceed, + ) + if trust_result.value not in successes: + raise ssl.SSLError( + "certificate verify failed, error code: %d" % + trust_result.value + ) + + def handshake( + self, + server_hostname, + verify, + trust_bundle, + min_version, + max_version, + client_cert, + client_key, + client_key_passphrase, + ): + """ + Actually performs the TLS handshake. This is run automatically by + wrapped socket, and shouldn't be needed in user code. + """ + # First, we do the initial bits of connection setup. We need to create + # a context, set its I/O funcs, and set the connection reference. + self.context = Security.SSLCreateContext( + None, SecurityConst.kSSLClientSide, SecurityConst.kSSLStreamType + ) + result = Security.SSLSetIOFuncs( + self.context, _read_callback_pointer, _write_callback_pointer + ) + _assert_no_error(result) + # Here we need to compute the handle to use. We do this by taking the + # id of self modulo 2**31 - 1. If this is already in the dictionary, we + # just keep incrementing by one until we find a free space. + with _connection_ref_lock: + handle = id(self) % 2147483647 + while handle in _connection_refs: + handle = (handle + 1) % 2147483647 + _connection_refs[handle] = self + result = Security.SSLSetConnection(self.context, handle) + _assert_no_error(result) + # If we have a server hostname, we should set that too. + if server_hostname: + if not isinstance(server_hostname, bytes): + server_hostname = server_hostname.encode('utf-8') + result = Security.SSLSetPeerDomainName( + self.context, server_hostname, len(server_hostname) + ) + _assert_no_error(result) + # Setup the ciphers. + self._set_ciphers() + # Set the minimum and maximum TLS versions. + result = Security.SSLSetProtocolVersionMin(self.context, min_version) + _assert_no_error(result) + result = Security.SSLSetProtocolVersionMax(self.context, max_version) + _assert_no_error(result) + # If there's a trust DB, we need to use it. We do that by telling + # SecureTransport to break on server auth. We also do that if we don't + # want to validate the certs at all: we just won't actually do any + # authing in that case. + if not verify or trust_bundle is not None: + result = Security.SSLSetSessionOption( + self.context, + SecurityConst.kSSLSessionOptionBreakOnServerAuth, + True, + ) + _assert_no_error(result) + # If there's a client cert, we need to use it. + if client_cert: + self._keychain, self._keychain_dir = _temporary_keychain() + self._client_cert_chain = _load_client_cert_chain( + self._keychain, client_cert, client_key + ) + result = Security.SSLSetCertificate( + self.context, self._client_cert_chain + ) + _assert_no_error(result) + while True: + with self._raise_on_error(): + result = Security.SSLHandshake(self.context) + if result == SecurityConst.errSSLWouldBlock: + raise socket.timeout("handshake timed out") + + elif result == SecurityConst.errSSLServerAuthCompleted: + self._custom_validate(verify, trust_bundle) + continue + + else: + _assert_no_error(result) + break + + def fileno(self): + return self.socket.fileno() + + + # Copy-pasted from Python 3.5 source code + def _decref_socketios(self): + if self._makefile_refs > 0: + self._makefile_refs -= 1 + if self._closed: + self.close() + + def recv(self, bufsiz): + buffer = ctypes.create_string_buffer(bufsiz) + bytes_read = self.recv_into(buffer, bufsiz) + data = buffer[:bytes_read] + return data + + def recv_into(self, buffer, nbytes=None): + # Read short on EOF. + if self._closed: + return 0 + + if nbytes is None: + nbytes = len(buffer) + buffer = (ctypes.c_char * nbytes).from_buffer(buffer) + processed_bytes = ctypes.c_size_t(0) + with self._raise_on_error(): + result = Security.SSLRead( + self.context, buffer, nbytes, ctypes.byref(processed_bytes) + ) + # There are some result codes that we want to treat as "not always + # errors". Specifically, those are errSSLWouldBlock, + # errSSLClosedGraceful, and errSSLClosedNoNotify. + if (result == SecurityConst.errSSLWouldBlock): + # If we didn't process any bytes, then this was just a time out. + # However, we can get errSSLWouldBlock in situations when we *did* + # read some data, and in those cases we should just read "short" + # and return. + if processed_bytes.value == 0: + # Timed out, no data read. + raise socket.timeout("recv timed out") + + elif result in ( + SecurityConst.errSSLClosedGraceful, + SecurityConst.errSSLClosedNoNotify, + ): + # The remote peer has closed this connection. We should do so as + # well. Note that we don't actually return here because in + # principle this could actually be fired along with return data. + # It's unlikely though. + self.close() + else: + _assert_no_error(result) + # Ok, we read and probably succeeded. We should return whatever data + # was actually read. + return processed_bytes.value + + def settimeout(self, timeout): + self._timeout = timeout + + def gettimeout(self): + return self._timeout + + def send(self, data): + processed_bytes = ctypes.c_size_t(0) + with self._raise_on_error(): + result = Security.SSLWrite( + self.context, data, len(data), ctypes.byref(processed_bytes) + ) + if result == SecurityConst.errSSLWouldBlock and processed_bytes.value == 0: + # Timed out + raise socket.timeout("send timed out") + + else: + _assert_no_error(result) + # We sent, and probably succeeded. Tell them how much we sent. + return processed_bytes.value + + def sendall(self, data): + total_sent = 0 + while total_sent < len(data): + sent = self.send(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE]) + total_sent += sent + + def shutdown(self): + with self._raise_on_error(): + Security.SSLClose(self.context) + + def close(self): + # TODO: should I do clean shutdown here? Do I have to? + if self._makefile_refs < 1: + self._closed = True + if self.context: + CoreFoundation.CFRelease(self.context) + self.context = None + if self._client_cert_chain: + CoreFoundation.CFRelease(self._client_cert_chain) + self._client_cert_chain = None + if self._keychain: + Security.SecKeychainDelete(self._keychain) + CoreFoundation.CFRelease(self._keychain) + shutil.rmtree(self._keychain_dir) + self._keychain = self._keychain_dir = None + return self.socket.close() + + else: + self._makefile_refs -= 1 + + def getpeercert(self, binary_form=False): + # Urgh, annoying. + # + # Here's how we do this: + # + # 1. Call SSLCopyPeerTrust to get hold of the trust object for this + # connection. + # 2. Call SecTrustGetCertificateAtIndex for index 0 to get the leaf. + # 3. To get the CN, call SecCertificateCopyCommonName and process that + # string so that it's of the appropriate type. + # 4. To get the SAN, we need to do something a bit more complex: + # a. Call SecCertificateCopyValues to get the data, requesting + # kSecOIDSubjectAltName. + # b. Mess about with this dictionary to try to get the SANs out. + # + # This is gross. Really gross. It's going to be a few hundred LoC extra + # just to repeat something that SecureTransport can *already do*. So my + # operating assumption at this time is that what we want to do is + # instead to just flag to urllib3 that it shouldn't do its own hostname + # validation when using SecureTransport. + if not binary_form: + raise ValueError( + "SecureTransport only supports dumping binary certs" + ) + + trust = Security.SecTrustRef() + certdata = None + der_bytes = None + try: + # Grab the trust store. + result = Security.SSLCopyPeerTrust( + self.context, ctypes.byref(trust) + ) + _assert_no_error(result) + if not trust: + # Probably we haven't done the handshake yet. No biggie. + return None + + cert_count = Security.SecTrustGetCertificateCount(trust) + if not cert_count: + # Also a case that might happen if we haven't handshaked. + # Handshook? Handshaken? + return None + + leaf = Security.SecTrustGetCertificateAtIndex(trust, 0) + assert leaf + # Ok, now we want the DER bytes. + certdata = Security.SecCertificateCopyData(leaf) + assert certdata + data_length = CoreFoundation.CFDataGetLength(certdata) + data_buffer = CoreFoundation.CFDataGetBytePtr(certdata) + der_bytes = ctypes.string_at(data_buffer, data_length) + finally: + if certdata: + CoreFoundation.CFRelease(certdata) + if trust: + CoreFoundation.CFRelease(trust) + return der_bytes + + def _reuse(self): + self._makefile_refs += 1 + + def _drop(self): + if self._makefile_refs < 1: + self.close() + else: + self._makefile_refs -= 1 + + +if _fileobject: # Platform-specific: Python 2 + + def makefile(self, mode, bufsize=-1): + self._makefile_refs += 1 + return _fileobject(self, mode, bufsize, close=True) + + +else: # Platform-specific: Python 3 + + def makefile(self, mode="r", buffering=None, *args, **kwargs): + # We disable buffering with SecureTransport because it conflicts with + # the buffering that ST does internally (see issue #1153 for more). + buffering = 0 + return backport_makefile(self, mode, buffering, *args, **kwargs) + + +WrappedSocket.makefile = makefile + + +class SecureTransportContext(object): + """ + I am a wrapper class for the SecureTransport library, to translate the + interface of the standard library ``SSLContext`` object to calls into + SecureTransport. + """ + + def __init__(self, protocol): + self._min_version, self._max_version = _protocol_to_min_max[protocol] + self._options = 0 + self._verify = False + self._trust_bundle = None + self._client_cert = None + self._client_key = None + self._client_key_passphrase = None + + @property + def check_hostname(self): + """ + SecureTransport cannot have its hostname checking disabled. For more, + see the comment on getpeercert() in this file. + """ + return True + + @check_hostname.setter + def check_hostname(self, value): + """ + SecureTransport cannot have its hostname checking disabled. For more, + see the comment on getpeercert() in this file. + """ + pass + + @property + def options(self): + # TODO: Well, crap. + # + # So this is the bit of the code that is the most likely to cause us + # trouble. Essentially we need to enumerate all of the SSL options that + # users might want to use and try to see if we can sensibly translate + # them, or whether we should just ignore them. + return self._options + + @options.setter + def options(self, value): + # TODO: Update in line with above. + self._options = value + + @property + def verify_mode(self): + return ssl.CERT_REQUIRED if self._verify else ssl.CERT_NONE + + @verify_mode.setter + def verify_mode(self, value): + self._verify = True if value == ssl.CERT_REQUIRED else False + + def set_default_verify_paths(self): + # So, this has to do something a bit weird. Specifically, what it does + # is nothing. + # + # This means that, if we had previously had load_verify_locations + # called, this does not undo that. We need to do that because it turns + # out that the rest of the urllib3 code will attempt to load the + # default verify paths if it hasn't been told about any paths, even if + # the context itself was sometime earlier. We resolve that by just + # ignoring it. + pass + + def load_default_certs(self): + return self.set_default_verify_paths() + + def set_ciphers(self, ciphers): + # For now, we just require the default cipher string. + if ciphers != util.ssl_.DEFAULT_CIPHERS: + raise ValueError( + "SecureTransport doesn't support custom cipher strings" + ) + + def load_verify_locations(self, cafile=None, capath=None, cadata=None): + # OK, we only really support cadata and cafile. + if capath is not None: + raise ValueError( + "SecureTransport does not support cert directories" + ) + + self._trust_bundle = cafile or cadata + + def load_cert_chain(self, certfile, keyfile=None, password=None): + self._client_cert = certfile + self._client_key = keyfile + self._client_cert_passphrase = password + + def wrap_socket( + self, + sock, + server_side=False, + do_handshake_on_connect=True, + suppress_ragged_eofs=True, + server_hostname=None, + ): + # So, what do we do here? Firstly, we assert some properties. This is a + # stripped down shim, so there is some functionality we don't support. + # See PEP 543 for the real deal. + assert not server_side + assert do_handshake_on_connect + assert suppress_ragged_eofs + # Ok, we're good to go. Now we want to create the wrapped socket object + # and store it in the appropriate place. + wrapped_socket = WrappedSocket(sock) + # Now we can handshake + wrapped_socket.handshake( + server_hostname, + self._verify, + self._trust_bundle, + self._min_version, + self._max_version, + self._client_cert, + self._client_key, + self._client_key_passphrase, + ) + return wrapped_socket diff --git a/requests3/core/_http/contrib/socks.py b/requests3/core/_http/contrib/socks.py new file mode 100644 index 00000000..bdabcb08 --- /dev/null +++ b/requests3/core/_http/contrib/socks.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +""" +This module contains provisional support for SOCKS proxies from within +urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and +SOCKS5. To enable its functionality, either install PySocks or install this +module with the ``socks`` extra. + +The SOCKS implementation supports the full range of urllib3 features. It also +supports the following SOCKS features: + +- SOCKS4 +- SOCKS4a +- SOCKS5 +- Usernames and passwords for the SOCKS proxy + +Known Limitations: + +- Currently PySocks does not support contacting remote websites via literal + IPv6 addresses. Any such connection attempt will fail. You must use a domain + name. +- Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any + such connection attempt will fail. +""" +from __future__ import absolute_import + +try: + import socks +except ImportError: + import warnings + from ..exceptions import DependencyWarning + + warnings.warn( + ( + 'SOCKS support in urllib3 requires the installation of optional ' + 'dependencies: specifically, PySocks. For more information, see ' + 'https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies' + ), + DependencyWarning, + ) + raise + +from socket import error as SocketError, timeout as SocketTimeout + +from .._sync.connection import (HTTP1Connection) +from ..connectionpool import (HTTPConnectionPool, HTTPSConnectionPool) +from ..exceptions import ConnectTimeoutError, NewConnectionError +from ..poolmanager import PoolManager +from ..util.url import parse_url + + +class SOCKSConnection(HTTP1Connection): + """ + A HTTP connection that connects via a SOCKS proxy. + """ + + def __init__(self, *args, **kwargs): + self._socks_options = kwargs.pop('_socks_options') + super(SOCKSConnection, self).__init__(*args, **kwargs) + + def _do_socket_connect(self, connect_timeout, connect_kw): + """ + Establish a new connection via the SOCKS proxy. + """ + try: + conn = socks.create_connection( + (self._host, self._port), + proxy_type=self._socks_options['socks_version'], + proxy_addr=self._socks_options['proxy_host'], + proxy_port=self._socks_options['proxy_port'], + proxy_username=self._socks_options['username'], + proxy_password=self._socks_options['password'], + proxy_rdns=self._socks_options['rdns'], + timeout=connect_timeout, + **connect_kw + ) + except SocketTimeout as e: + raise ConnectTimeoutError( + self, + "Connection to %s timed out. (connect timeout=%s)" % + (self._host, connect_timeout), + ) + + except socks.ProxyError as e: + # This is fragile as hell, but it seems to be the only way to raise + # useful errors here. + if e.socket_err: + error = e.socket_err + if isinstance(error, SocketTimeout): + raise ConnectTimeoutError( + self, + "Connection to %s timed out. (connect timeout=%s)" % + (self._host, connect_timeout), + ) + + else: + raise NewConnectionError( + self, + "Failed to establish a new connection: %s" % error, + ) + + else: + raise NewConnectionError( + self, "Failed to establish a new connection: %s" % e + ) + + except SocketError as e: # Defensive: PySocks should catch all these. + raise NewConnectionError( + self, "Failed to establish a new connection: %s" % e + ) + + return conn + + +class SOCKSHTTPConnectionPool(HTTPConnectionPool): + ConnectionCls = SOCKSConnection + + +class SOCKSHTTPSConnectionPool(HTTPSConnectionPool): + ConnectionCls = SOCKSConnection + + +class SOCKSProxyManager(PoolManager): + """ + A version of the urllib3 ProxyManager that routes connections via the + defined SOCKS proxy. + """ + pool_classes_by_scheme = { + 'http': SOCKSHTTPConnectionPool, 'https': SOCKSHTTPSConnectionPool + } + + def __init__( + self, + proxy_url, + username=None, + password=None, + num_pools=10, + headers=None, + **connection_pool_kw + ): + parsed = parse_url(proxy_url) + if parsed.scheme == 'socks5': + socks_version = socks.PROXY_TYPE_SOCKS5 + rdns = False + elif parsed.scheme == 'socks5h': + socks_version = socks.PROXY_TYPE_SOCKS5 + rdns = True + elif parsed.scheme == 'socks4': + socks_version = socks.PROXY_TYPE_SOCKS4 + rdns = False + elif parsed.scheme == 'socks4a': + socks_version = socks.PROXY_TYPE_SOCKS4 + rdns = True + else: + raise ValueError( + "Unable to determine SOCKS version from %s" % proxy_url + ) + + self.proxy_url = proxy_url + socks_options = { + 'socks_version': socks_version, + 'proxy_host': parsed.host, + 'proxy_port': parsed.port, + 'username': username, + 'password': password, + 'rdns': rdns, + } + connection_pool_kw['_socks_options'] = socks_options + super(SOCKSProxyManager, self).__init__( + num_pools, headers, **connection_pool_kw + ) + self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme diff --git a/requests3/core/_http/exceptions.py b/requests3/core/_http/exceptions.py new file mode 100644 index 00000000..743a6927 --- /dev/null +++ b/requests3/core/_http/exceptions.py @@ -0,0 +1,238 @@ +from __future__ import absolute_import + + + +# Base Exceptions +class HTTPError(Exception): + "Base exception used by this module." + pass + + +class HTTPWarning(Warning): + "Base warning used by this module." + pass + + +class PoolError(HTTPError): + "Base exception for errors caused within a pool." + + def __init__(self, pool, message): + self.pool = pool + HTTPError.__init__(self, "%s: %s" % (pool, message)) + + def __reduce__(self): + # For pickling purposes. + return self.__class__, (None, None) + + +class RequestError(PoolError): + "Base exception for PoolErrors that have associated URLs." + + def __init__(self, pool, url, message): + self.url = url + PoolError.__init__(self, pool, message) + + def __reduce__(self): + # For pickling purposes. + return self.__class__, (None, self.url, None) + + +class SSLError(HTTPError): + "Raised when SSL certificate fails in an HTTPS connection." + pass + + +class ProxyError(HTTPError): + "Raised when the connection to a proxy fails." + pass + + +class DecodeError(HTTPError): + "Raised when automatic decoding based on Content-Type fails." + pass + + +class ProtocolError(HTTPError): + "Raised when something unexpected happens mid-request/response." + pass + + +# : Renamed to ProtocolError but aliased for backwards compatibility. +ConnectionError = ProtocolError + + +# Leaf Exceptions +class MaxRetryError(RequestError): + """Raised when the maximum number of retries is exceeded. + + :param pool: The connection pool + :type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool` + :param string url: The requested Url + :param exceptions.Exception reason: The underlying error + + """ + + def __init__(self, pool, url, reason=None): + self.reason = reason + message = "Max retries exceeded with url: %s (Caused by %r)" % ( + url, reason + ) + RequestError.__init__(self, pool, url, message) + + +class TimeoutStateError(HTTPError): + """ Raised when passing an invalid state to a timeout """ + pass + + +class TimeoutError(HTTPError): + """ Raised when a socket timeout error occurs. + + Catching this error will catch both :exc:`ReadTimeoutErrors + ` and :exc:`ConnectTimeoutErrors `. + """ + pass + + +class ReadTimeoutError(TimeoutError, RequestError): + "Raised when a socket timeout occurs while receiving data from a server" + pass + + + + +# This timeout error does not have a URL attached and needs to inherit from the +# base HTTPError +class ConnectTimeoutError(TimeoutError): + "Raised when a socket timeout occurs while connecting to a server" + pass + + +class NewConnectionError(ConnectTimeoutError, PoolError): + "Raised when we fail to establish a new connection. Usually ECONNREFUSED." + pass + + +class EmptyPoolError(PoolError): + "Raised when a pool runs out of connections and no more are allowed." + pass + + +class ClosedPoolError(PoolError): + "Raised when a request enters a pool after the pool has been closed." + pass + + +class LocationValueError(ValueError, HTTPError): + "Raised when there is something wrong with a given URL input." + pass + + +class LocationParseError(LocationValueError): + "Raised when get_host or similar fails to parse the URL input." + + def __init__(self, location): + message = "Failed to parse: %s" % location + HTTPError.__init__(self, message) + self.location = location + + +class ResponseError(HTTPError): + "Used as a container for an error reason supplied in a MaxRetryError." + GENERIC_ERROR = 'too many error responses' + SPECIFIC_ERROR = 'too many {status_code} error responses' + + +class SecurityWarning(HTTPWarning): + "Warned when perfoming security reducing actions" + pass + + +class SubjectAltNameWarning(SecurityWarning): + "Warned when connecting to a host with a certificate missing a SAN." + pass + + +class InsecureRequestWarning(SecurityWarning): + "Warned when making an unverified HTTPS request." + pass + + +class SystemTimeWarning(SecurityWarning): + "Warned when system time is suspected to be wrong" + pass + + +class InsecurePlatformWarning(SecurityWarning): + "Warned when certain SSL configuration is not available on a platform." + pass + + +class SNIMissingWarning(HTTPWarning): + "Warned when making a HTTPS request without SNI available." + pass + + +class DependencyWarning(HTTPWarning): + """ + Warned when an attempt is made to import a module with missing optional + dependencies. + """ + pass + + +class InvalidHeader(HTTPError): + "The header provided was somehow invalid." + pass + + +class BadVersionError(ProtocolError): + """ + The HTTP version in the response is unsupported. + """ + + def __init__(self, version): + message = "HTTP version {} is unsupported".format(version) + super(BadVersionError, self).__init__(message) + + +class ProxySchemeUnknown(AssertionError, ValueError): + "ProxyManager does not support the supplied scheme" + + # TODO(t-8ch): Stop inheriting from AssertionError in v2.0. + def __init__(self, scheme): + message = "Not supported proxy scheme %s" % scheme + super(ProxySchemeUnknown, self).__init__(message) + + +class HeaderParsingError(HTTPError): + "Raised by assert_header_parsing, but we convert it to a log.warning statement." + + def __init__(self, defects, unparsed_data): + message = '%s, unparsed data: %r' % ( + defects or 'Unknown', unparsed_data + ) + super(HeaderParsingError, self).__init__(message) + + +class UnrewindableBodyError(HTTPError): + "urllib3 encountered an error when trying to rewind a body" + pass + + +class FailedTunnelError(HTTPError): + """ + An attempt was made to set up a CONNECT tunnel, but that attempt failed. + """ + + def __init__(self, message, response): + super(FailedTunnelError, self).__init__(message) + self.response = response + + +class InvalidBodyError(HTTPError): + """ + An attempt was made to send a request with a body object that urllib3 does + not support. + """ + pass diff --git a/requests3/core/_http/fields.py b/requests3/core/_http/fields.py new file mode 100644 index 00000000..f1808f0e --- /dev/null +++ b/requests3/core/_http/fields.py @@ -0,0 +1,183 @@ +from __future__ import absolute_import +import email.utils +import mimetypes + +from .packages import six + + +def guess_content_type(filename, default='application/octet-stream'): + """ + Guess the "Content-Type" of a file. + + :param filename: + The filename to guess the "Content-Type" of using :mod:`mimetypes`. + :param default: + If no "Content-Type" can be guessed, default to `default`. + """ + if filename: + return mimetypes.guess_type(filename)[0] or default + + return default + + +def format_header_param(name, value): + """ + Helper function to format and quote a single header parameter. + + Particularly useful for header parameters which might contain + non-ASCII values, like file names. This follows RFC 2231, as + suggested by RFC 2388 Section 4.4. + + :param name: + The name of the parameter, a string expected to be ASCII only. + :param value: + The value of the parameter, provided as a unicode string. + """ + if not any(ch in value for ch in '"\\\r\n'): + result = '%s="%s"' % (name, value) + try: + result.encode('ascii') + except (UnicodeEncodeError, UnicodeDecodeError): + pass + else: + return result + + if not six.PY3 and isinstance(value, six.text_type): # Python 2: + value = value.encode('utf-8') + value = email.utils.encode_rfc2231(value, 'utf-8') + value = '%s*=%s' % (name, value) + return value + + +class RequestField(object): + """ + A data container for request body parameters. + + :param name: + The name of this request field. + :param data: + The data/value body. + :param filename: + An optional filename of the request field. + :param headers: + An optional dict-like object of headers to initially use for the field. + """ + + def __init__(self, name, data, filename=None, headers=None): + self._name = name + self._filename = filename + self.data = data + self.headers = {} + if headers: + self.headers = dict(headers) + + @classmethod + def from_tuples(cls, fieldname, value): + """ + A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters. + + Supports constructing :class:`~urllib3.fields.RequestField` from + parameter of key/value strings AND key/filetuple. A filetuple is a + (filename, data, MIME type) tuple where the MIME type is optional. + For example:: + + 'foo': 'bar', + 'fakefile': ('foofile.txt', 'contents of foofile'), + 'realfile': ('barfile.txt', open('realfile').read()), + 'typedfile': ('bazfile.bin', open('bazfile').read(), 'image/jpeg'), + 'nonamefile': 'contents of nonamefile field', + + Field names and filenames must be unicode. + """ + if isinstance(value, tuple): + if len(value) == 3: + filename, data, content_type = value + else: + filename, data = value + content_type = guess_content_type(filename) + else: + filename = None + content_type = None + data = value + request_param = cls(fieldname, data, filename=filename) + request_param.make_multipart(content_type=content_type) + return request_param + + def _render_part(self, name, value): + """ + Overridable helper function to format a single header parameter. + + :param name: + The name of the parameter, a string expected to be ASCII only. + :param value: + The value of the parameter, provided as a unicode string. + """ + return format_header_param(name, value) + + def _render_parts(self, header_parts): + """ + Helper function to format and quote a single header. + + Useful for single headers that are composed of multiple items. E.g., + 'Content-Disposition' fields. + + :param header_parts: + A sequence of (k, v) typles or a :class:`dict` of (k, v) to format + as `k1="v1"; k2="v2"; ...`. + """ + parts = [] + iterable = header_parts + if isinstance(header_parts, dict): + iterable = header_parts.items() + for name, value in iterable: + if value is not None: + parts.append(self._render_part(name, value)) + return '; '.join(parts) + + def render_headers(self): + """ + Renders the headers for this request field. + """ + lines = [] + sort_keys = ['Content-Disposition', 'Content-Type', 'Content-Location'] + for sort_key in sort_keys: + if self.headers.get(sort_key, False): + lines.append('%s: %s' % (sort_key, self.headers[sort_key])) + for header_name, header_value in self.headers.items(): + if header_name not in sort_keys: + if header_value: + lines.append('%s: %s' % (header_name, header_value)) + lines.append('\r\n') + return '\r\n'.join(lines) + + def make_multipart( + self, + content_disposition=None, + content_type=None, + content_location=None, + ): + """ + Makes this request field into a multipart request field. + + This method overrides "Content-Disposition", "Content-Type" and + "Content-Location" headers to the request parameter. + + :param content_type: + The 'Content-Type' of the request body. + :param content_location: + The 'Content-Location' of the request body. + + """ + self.headers[ + 'Content-Disposition' + ] = content_disposition or 'form-data' + self.headers['Content-Disposition'] += '; '.join( + [ + '', + self._render_parts( + (('name', self._name), ('filename', self._filename)) + ), + ] + ) + self.headers['Content-Type'] = content_type + self.headers['Content-Location'] = content_location diff --git a/requests3/core/_http/filepost.py b/requests3/core/_http/filepost.py new file mode 100644 index 00000000..6b05b747 --- /dev/null +++ b/requests3/core/_http/filepost.py @@ -0,0 +1,93 @@ +from __future__ import absolute_import +import codecs + +from io import BytesIO + +from .packages import six +from .packages.six import b +from .fields import RequestField + +writer = codecs.lookup('utf-8')[3] + + +def choose_boundary(): + """ + Our embarrassingly-simple replacement for mimetools.choose_boundary. + + We are lazily loading uuid here, because we don't want its issues + + https://bugs.python.org/issue5885 + https://bugs.python.org/issue11063 + + to affect our entire library. + """ + from uuid import uuid4 + return uuid4().hex + + +def iter_field_objects(fields): + """ + Iterate over fields. + + Supports list of (k, v) tuples and dicts, and lists of + :class:`~urllib3.fields.RequestField`. + + """ + if isinstance(fields, dict): + i = six.iteritems(fields) + else: + i = iter(fields) + for field in i: + if isinstance(field, RequestField): + yield field + + else: + yield RequestField.from_tuples(*field) + + +def iter_fields(fields): + """ + .. deprecated:: 1.6 + + Iterate over fields. + + The addition of :class:`~urllib3.fields.RequestField` makes this function + obsolete. Instead, use :func:`iter_field_objects`, which returns + :class:`~urllib3.fields.RequestField` objects. + + Supports list of (k, v) tuples and dicts. + """ + if isinstance(fields, dict): + return ((k, v) for k, v in six.iteritems(fields)) + + return ((k, v) for k, v in fields) + + +def encode_multipart_formdata(fields, boundary=None): + """ + Encode a dictionary of ``fields`` using the multipart/form-data MIME format. + + :param fields: + Dictionary of fields or list of (key, :class:`~urllib3.fields.RequestField`). + + :param boundary: + If not specified, then a random boundary will be generated using + :func:`mimetools.choose_boundary`. + """ + body = BytesIO() + if boundary is None: + boundary = choose_boundary() + for field in iter_field_objects(fields): + body.write(b('--%s\r\n' % (boundary))) + writer(body).write(field.render_headers()) + data = field.data + if isinstance(data, int): + data = str(data) # Backwards compatibility + if isinstance(data, six.text_type): + writer(body).write(data) + else: + body.write(data) + body.write(b'\r\n') + body.write(b('--%s--\r\n' % (boundary))) + content_type = str('multipart/form-data; boundary=%s' % boundary) + return body.getvalue(), content_type diff --git a/requests3/core/_http/packages/__init__.py b/requests3/core/_http/packages/__init__.py new file mode 100644 index 00000000..b3e85f85 --- /dev/null +++ b/requests3/core/_http/packages/__init__.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import + +from . import ssl_match_hostname + +__all__ = ('ssl_match_hostname',) diff --git a/requests3/core/_http/packages/backports/__init__.py b/requests3/core/_http/packages/backports/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/requests3/core/_http/packages/backports/makefile.py b/requests3/core/_http/packages/backports/makefile.py new file mode 100644 index 00000000..160f0666 --- /dev/null +++ b/requests3/core/_http/packages/backports/makefile.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +""" +backports.makefile +~~~~~~~~~~~~~~~~~~ + +Backports the Python 3 ``socket.makefile`` method for use with anything that +wants to create a "fake" socket object. +""" +import io + +from socket import SocketIO + + +def backport_makefile( + self, mode="r", buffering=None, encoding=None, errors=None, newline=None +): + """ + Backport of ``socket.makefile`` from Python 3.5. + """ + if not set(mode) <= set(["r", "w", "b"]): + raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,)) + + writing = "w" in mode + reading = "r" in mode or not writing + assert reading or writing + binary = "b" in mode + rawmode = "" + if reading: + rawmode += "r" + if writing: + rawmode += "w" + raw = SocketIO(self, rawmode) + self._makefile_refs += 1 + if buffering is None: + buffering = -1 + if buffering < 0: + buffering = io.DEFAULT_BUFFER_SIZE + if buffering == 0: + if not binary: + raise ValueError("unbuffered streams must be binary") + + return raw + + if reading and writing: + buffer = io.BufferedRWPair(raw, raw, buffering) + elif reading: + buffer = io.BufferedReader(raw, buffering) + else: + assert writing + buffer = io.BufferedWriter(raw, buffering) + if binary: + return buffer + + text = io.TextIOWrapper(buffer, encoding, errors, newline) + text.mode = mode + return text diff --git a/requests3/core/_http/packages/ordered_dict.py b/requests3/core/_http/packages/ordered_dict.py new file mode 100644 index 00000000..74845861 --- /dev/null +++ b/requests3/core/_http/packages/ordered_dict.py @@ -0,0 +1,272 @@ +# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. +# Passes Python2.7's test suite and incorporates all the latest updates. +# Copyright 2009 Raymond Hettinger, released under the MIT License. +# http://code.activestate.com/recipes/576693/ +try: + from thread import get_ident as _get_ident +except ImportError: + from dummy_thread import get_ident as _get_ident +try: + from _abcoll import KeysView, ValuesView, ItemsView +except ImportError: + pass + + +class OrderedDict(dict): + 'Dictionary that remembers insertion order' + + # An inherited dict maps keys to values. + # The inherited dict provides __getitem__, __len__, __contains__, and get. + # The remaining methods are order-aware. + # Big-O running times for all methods are the same as for regular dictionaries. + # The internal self.__map dictionary maps keys to links in a doubly linked list. + # The circular doubly linked list starts and ends with a sentinel element. + # The sentinel element never gets deleted (this simplifies the algorithm). + # Each link is stored as a list of length three: [PREV, NEXT, KEY]. + def __init__(self, *args, **kwds): + '''Initialize an ordered dictionary. Signature is the same as for + regular dictionaries, but keyword arguments are not recommended + because their insertion order is arbitrary. + + ''' + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + + try: + self.__root + except AttributeError: + self.__root = root = [] # sentinel node + root[:] = [root, root, None] + self.__map = {} + self.__update(*args, **kwds) + + def __setitem__(self, key, value, dict_setitem=dict.__setitem__): + 'od.__setitem__(i, y) <==> od[i]=y' + # Setting a new item creates a new link which goes at the end of the linked + # list, and the inherited dictionary is updated with the new key/value pair. + if key not in self: + root = self.__root + last = root[0] + last[1] = root[0] = self.__map[key] = [last, root, key] + dict_setitem(self, key, value) + + def __delitem__(self, key, dict_delitem=dict.__delitem__): + 'od.__delitem__(y) <==> del od[y]' + # Deleting an existing item uses self.__map to find the link which is + # then removed by updating the links in the predecessor and successor nodes. + dict_delitem(self, key) + link_prev, link_next, key = self.__map.pop(key) + link_prev[1] = link_next + link_next[0] = link_prev + + def __iter__(self): + 'od.__iter__() <==> iter(od)' + root = self.__root + curr = root[1] + while curr is not root: + yield curr[2] + + curr = curr[1] + + def __reversed__(self): + 'od.__reversed__() <==> reversed(od)' + root = self.__root + curr = root[0] + while curr is not root: + yield curr[2] + + curr = curr[0] + + def clear(self): + 'od.clear() -> None. Remove all items from od.' + try: + for node in self.__map.itervalues(): + del node[:] + root = self.__root + root[:] = [root, root, None] + self.__map.clear() + except AttributeError: + pass + dict.clear(self) + + def popitem(self, last=True): + '''od.popitem() -> (k, v), return and remove a (key, value) pair. + Pairs are returned in LIFO order if last is true or FIFO order if false. + + ''' + if not self: + raise KeyError('dictionary is empty') + + root = self.__root + if last: + link = root[0] + link_prev = link[0] + link_prev[1] = root + root[0] = link_prev + else: + link = root[1] + link_next = link[1] + root[1] = link_next + link_next[0] = root + key = link[2] + del self.__map[key] + value = dict.pop(self, key) + return key, value + + + # -- the following methods do not depend on the internal structure -- + def keys(self): + 'od.keys() -> list of keys in od' + return list(self) + + def values(self): + 'od.values() -> list of values in od' + return [self[key] for key in self] + + def items(self): + 'od.items() -> list of (key, value) pairs in od' + return [(key, self[key]) for key in self] + + def iterkeys(self): + 'od.iterkeys() -> an iterator over the keys in od' + return iter(self) + + def itervalues(self): + 'od.itervalues -> an iterator over the values in od' + for k in self: + yield self[k] + + def iteritems(self): + 'od.iteritems -> an iterator over the (key, value) items in od' + for k in self: + yield (k, self[k]) + + def update(*args, **kwds): + '''od.update(E, **F) -> None. Update od from dict/iterable E and F. + + If E is a dict instance, does: for k in E: od[k] = E[k] + If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] + Or if E is an iterable of items, does: for k, v in E: od[k] = v + In either case, this is followed by: for k, v in F.items(): od[k] = v + + ''' + if len(args) > 2: + raise TypeError( + 'update() takes at most 2 positional ' + 'arguments (%d given)' % (len(args),) + ) + + elif not args: + raise TypeError('update() takes at least 1 argument (0 given)') + + self = args[0] + # Make progressively weaker assumptions about "other" + other = () + if len(args) == 2: + other = args[1] + if isinstance(other, dict): + for key in other: + self[key] = other[key] + elif hasattr(other, 'keys'): + for key in other.keys(): + self[key] = other[key] + else: + for key, value in other: + self[key] = value + for key, value in kwds.items(): + self[key] = value + + __update = update # let subclasses override update without breaking __init__ + __marker = object() + + def pop(self, key, default=__marker): + '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. + If key is not found, d is returned if given, otherwise KeyError is raised. + + ''' + if key in self: + result = self[key] + del self[key] + return result + + if default is self.__marker: + raise KeyError(key) + + return default + + def setdefault(self, key, default=None): + 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' + if key in self: + return self[key] + + self[key] = default + return default + + def __repr__(self, _repr_running={}): + 'od.__repr__() <==> repr(od)' + call_key = id(self), _get_ident() + if call_key in _repr_running: + return '...' + + _repr_running[call_key] = 1 + try: + if not self: + return '%s()' % (self.__class__.__name__,) + + return '%s(%r)' % (self.__class__.__name__, self.items()) + + finally: + del _repr_running[call_key] + + def __reduce__(self): + 'Return state information for pickling' + items = [[k, self[k]] for k in self] + inst_dict = vars(self).copy() + for k in vars(OrderedDict()): + inst_dict.pop(k, None) + if inst_dict: + return (self.__class__, (items,), inst_dict) + + return self.__class__, (items,) + + def copy(self): + 'od.copy() -> a shallow copy of od' + return self.__class__(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S + and values equal to v (which defaults to None). + + ''' + d = cls() + for key in iterable: + d[key] = value + return d + + def __eq__(self, other): + '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive + while comparison to a regular mapping is order-insensitive. + + ''' + if isinstance(other, OrderedDict): + return len(self) == len(other) and self.items() == other.items() + + return dict.__eq__(self, other) + + def __ne__(self, other): + return not self == other + + + # -- the following methods are only used in Python 2.7 -- + def viewkeys(self): + "od.viewkeys() -> a set-like object providing a view on od's keys" + return KeysView(self) + + def viewvalues(self): + "od.viewvalues() -> an object providing a view on od's values" + return ValuesView(self) + + def viewitems(self): + "od.viewitems() -> a set-like object providing a view on od's items" + return ItemsView(self) diff --git a/requests3/core/_http/packages/six.py b/requests3/core/_http/packages/six.py new file mode 100644 index 00000000..af378941 --- /dev/null +++ b/requests3/core/_http/packages/six.py @@ -0,0 +1,935 @@ +"""Utilities for writing code that runs on Python 2 and 3""" +# Copyright (c) 2010-2015 Benjamin Peterson +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +from __future__ import absolute_import + +import functools +import itertools +import operator +import sys +import types + +__author__ = "Benjamin Peterson " +__version__ = "1.10.0" +# Useful for very coarse version differentiation. +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 +PY34 = sys.version_info[0:2] >= (3, 4) +if PY3: + string_types = str, + integer_types = int, + class_types = type, + text_type = str + binary_type = bytes + MAXSIZE = sys.maxsize +else: + string_types = basestring, + integer_types = (int, long) + class_types = (type, types.ClassType) + text_type = unicode + binary_type = str + if sys.platform.startswith("java"): + # Jython always uses 32 bits. + MAXSIZE = int((1 << 31) - 1) + else: + + # It's possible to have sizeof(long) != sizeof(Py_ssize_t). + class X(object): + + def __len__(self): + return 1 << 31 + + try: + len(X()) + except OverflowError: + # 32-bit + MAXSIZE = int((1 << 31) - 1) + else: + # 64-bit + MAXSIZE = int((1 << 63) - 1) + del X + + +def _add_doc(func, doc): + """Add documentation to a function.""" + func.__doc__ = doc + + +def _import_module(name): + """Import module, returning the module after the last dot.""" + __import__(name) + return sys.modules[name] + + +class _LazyDescr(object): + + def __init__(self, name): + self.name = name + + def __get__(self, obj, tp): + result = self._resolve() + setattr(obj, self.name, result) # Invokes __set__. + try: + # This is a bit ugly, but it avoids running this again by + # removing this descriptor. + delattr(obj.__class__, self.name) + except AttributeError: + pass + return result + + +class MovedModule(_LazyDescr): + + def __init__(self, name, old, new=None): + super(MovedModule, self).__init__(name) + if PY3: + if new is None: + new = name + self.mod = new + else: + self.mod = old + + def _resolve(self): + return _import_module(self.mod) + + def __getattr__(self, attr): + _module = self._resolve() + value = getattr(_module, attr) + setattr(self, attr, value) + return value + + +class _LazyModule(types.ModuleType): + + def __init__(self, name): + super(_LazyModule, self).__init__(name) + self.__doc__ = self.__class__.__doc__ + + def __dir__(self): + attrs = ["__doc__", "__name__"] + attrs += [attr.name for attr in self._moved_attributes] + return attrs + + # Subclasses should override this + _moved_attributes = [] + + +class MovedAttribute(_LazyDescr): + + def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): + super(MovedAttribute, self).__init__(name) + if PY3: + if new_mod is None: + new_mod = name + self.mod = new_mod + if new_attr is None: + if old_attr is None: + new_attr = name + else: + new_attr = old_attr + self.attr = new_attr + else: + self.mod = old_mod + if old_attr is None: + old_attr = name + self.attr = old_attr + + def _resolve(self): + module = _import_module(self.mod) + return getattr(module, self.attr) + + +class _SixMetaPathImporter(object): + """ + A meta path importer to import six.moves and its submodules. + + This class implements a PEP302 finder and loader. It should be compatible + with Python 2.5 and all existing versions of Python3 + """ + + def __init__(self, six_module_name): + self.name = six_module_name + self.known_modules = {} + + def _add_module(self, mod, *fullnames): + for fullname in fullnames: + self.known_modules[self.name + "." + fullname] = mod + + def _get_module(self, fullname): + return self.known_modules[self.name + "." + fullname] + + def find_module(self, fullname, path=None): + if fullname in self.known_modules: + return self + + return None + + def __get_module(self, fullname): + try: + return self.known_modules[fullname] + + except KeyError: + raise ImportError("This loader does not know module " + fullname) + + def load_module(self, fullname): + try: + # in case of a reload + return sys.modules[fullname] + + except KeyError: + pass + mod = self.__get_module(fullname) + if isinstance(mod, MovedModule): + mod = mod._resolve() + else: + mod.__loader__ = self + sys.modules[fullname] = mod + return mod + + def is_package(self, fullname): + """ + Return true, if the named module is a package. + + We need this method to get correct spec objects with + Python 3.4 (see PEP451) + """ + return hasattr(self.__get_module(fullname), "__path__") + + def get_code(self, fullname): + """Return None + + Required, if is_package is implemented""" + self.__get_module(fullname) # eventually raises ImportError + return None + + get_source = get_code # same as get_code + + +_importer = _SixMetaPathImporter(__name__) + + +class _MovedItems(_LazyModule): + """Lazy loading of moved objects""" + __path__ = [] # mark as package + + +_moved_attributes = [ + MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), + MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), + MovedAttribute( + "filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse" + ), + MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), + MovedAttribute("intern", "__builtin__", "sys"), + MovedAttribute("map", "itertools", "builtins", "imap", "map"), + MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), + MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute( + "reload_module", + "__builtin__", + "importlib" if PY34 else "imp", + "reload", + ), + MovedAttribute("reduce", "__builtin__", "functools"), + MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), + MovedAttribute("StringIO", "StringIO", "io"), + MovedAttribute("UserDict", "UserDict", "collections"), + MovedAttribute("UserList", "UserList", "collections"), + MovedAttribute("UserString", "UserString", "collections"), + MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), + MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), + MovedAttribute( + "zip_longest", "itertools", "itertools", "izip_longest", "zip_longest" + ), + MovedModule("builtins", "__builtin__"), + MovedModule("configparser", "ConfigParser"), + MovedModule("copyreg", "copy_reg"), + MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), + MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), + MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), + MovedModule("http_cookies", "Cookie", "http.cookies"), + MovedModule("html_entities", "htmlentitydefs", "html.entities"), + MovedModule("html_parser", "HTMLParser", "html.parser"), + MovedModule("http_client", "httplib", "http.client"), + MovedModule( + "email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart" + ), + MovedModule( + "email_mime_nonmultipart", + "email.MIMENonMultipart", + "email.mime.nonmultipart", + ), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), + MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), + MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), + MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), + MovedModule("cPickle", "cPickle", "pickle"), + MovedModule("queue", "Queue"), + MovedModule("reprlib", "repr"), + MovedModule("socketserver", "SocketServer"), + MovedModule("_thread", "thread", "_thread"), + MovedModule("tkinter", "Tkinter"), + MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), + MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), + MovedModule( + "tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext" + ), + MovedModule( + "tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog" + ), + MovedModule("tkinter_tix", "Tix", "tkinter.tix"), + MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), + MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), + MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), + MovedModule( + "tkinter_colorchooser", "tkColorChooser", "tkinter.colorchooser" + ), + MovedModule( + "tkinter_commondialog", "tkCommonDialog", "tkinter.commondialog" + ), + MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), + MovedModule("tkinter_font", "tkFont", "tkinter.font"), + MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), + MovedModule( + "tkinter_tksimpledialog", "tkSimpleDialog", "tkinter.simpledialog" + ), + MovedModule( + "urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse" + ), + MovedModule( + "urllib_error", __name__ + ".moves.urllib_error", "urllib.error" + ), + MovedModule( + "urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib" + ), + MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), + MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), + MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), +] +# Add windows specific modules. +if sys.platform == "win32": + _moved_attributes += [MovedModule("winreg", "_winreg")] +for attr in _moved_attributes: + setattr(_MovedItems, attr.name, attr) + if isinstance(attr, MovedModule): + _importer._add_module(attr, "moves." + attr.name) +del attr +_MovedItems._moved_attributes = _moved_attributes +moves = _MovedItems(__name__ + ".moves") +_importer._add_module(moves, "moves") + + +class Module_six_moves_urllib_parse(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_parse""" + + +_urllib_parse_moved_attributes = [ + MovedAttribute("ParseResult", "urlparse", "urllib.parse"), + MovedAttribute("SplitResult", "urlparse", "urllib.parse"), + MovedAttribute("parse_qs", "urlparse", "urllib.parse"), + MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), + MovedAttribute("urldefrag", "urlparse", "urllib.parse"), + MovedAttribute("urljoin", "urlparse", "urllib.parse"), + MovedAttribute("urlparse", "urlparse", "urllib.parse"), + MovedAttribute("urlsplit", "urlparse", "urllib.parse"), + MovedAttribute("urlunparse", "urlparse", "urllib.parse"), + MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), + MovedAttribute("quote", "urllib", "urllib.parse"), + MovedAttribute("quote_plus", "urllib", "urllib.parse"), + MovedAttribute("unquote", "urllib", "urllib.parse"), + MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute("urlencode", "urllib", "urllib.parse"), + MovedAttribute("splitquery", "urllib", "urllib.parse"), + MovedAttribute("splittag", "urllib", "urllib.parse"), + MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), + MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), + MovedAttribute("uses_params", "urlparse", "urllib.parse"), + MovedAttribute("uses_query", "urlparse", "urllib.parse"), + MovedAttribute("uses_relative", "urlparse", "urllib.parse"), +] +for attr in _urllib_parse_moved_attributes: + setattr(Module_six_moves_urllib_parse, attr.name, attr) +del attr +Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes +_importer._add_module( + Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), + "moves.urllib_parse", + "moves.urllib.parse", +) + + +class Module_six_moves_urllib_error(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_error""" + + +_urllib_error_moved_attributes = [ + MovedAttribute("URLError", "urllib2", "urllib.error"), + MovedAttribute("HTTPError", "urllib2", "urllib.error"), + MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), +] +for attr in _urllib_error_moved_attributes: + setattr(Module_six_moves_urllib_error, attr.name, attr) +del attr +Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes +_importer._add_module( + Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), + "moves.urllib_error", + "moves.urllib.error", +) + + +class Module_six_moves_urllib_request(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_request""" + + +_urllib_request_moved_attributes = [ + MovedAttribute("urlopen", "urllib2", "urllib.request"), + MovedAttribute("install_opener", "urllib2", "urllib.request"), + MovedAttribute("build_opener", "urllib2", "urllib.request"), + MovedAttribute("pathname2url", "urllib", "urllib.request"), + MovedAttribute("url2pathname", "urllib", "urllib.request"), + MovedAttribute("getproxies", "urllib", "urllib.request"), + MovedAttribute("Request", "urllib2", "urllib.request"), + MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), + MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), + MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), + MovedAttribute("BaseHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), + MovedAttribute( + "HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request" + ), + MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), + MovedAttribute("FileHandler", "urllib2", "urllib.request"), + MovedAttribute("FTPHandler", "urllib2", "urllib.request"), + MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), + MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), + MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), + MovedAttribute("urlretrieve", "urllib", "urllib.request"), + MovedAttribute("urlcleanup", "urllib", "urllib.request"), + MovedAttribute("URLopener", "urllib", "urllib.request"), + MovedAttribute("FancyURLopener", "urllib", "urllib.request"), + MovedAttribute("proxy_bypass", "urllib", "urllib.request"), +] +for attr in _urllib_request_moved_attributes: + setattr(Module_six_moves_urllib_request, attr.name, attr) +del attr +Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes +_importer._add_module( + Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), + "moves.urllib_request", + "moves.urllib.request", +) + + +class Module_six_moves_urllib_response(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_response""" + + +_urllib_response_moved_attributes = [ + MovedAttribute("addbase", "urllib", "urllib.response"), + MovedAttribute("addclosehook", "urllib", "urllib.response"), + MovedAttribute("addinfo", "urllib", "urllib.response"), + MovedAttribute("addinfourl", "urllib", "urllib.response"), +] +for attr in _urllib_response_moved_attributes: + setattr(Module_six_moves_urllib_response, attr.name, attr) +del attr +Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes +_importer._add_module( + Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), + "moves.urllib_response", + "moves.urllib.response", +) + + +class Module_six_moves_urllib_robotparser(_LazyModule): + """Lazy loading of moved objects in six.moves.urllib_robotparser""" + + +_urllib_robotparser_moved_attributes = [ + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser") +] +for attr in _urllib_robotparser_moved_attributes: + setattr(Module_six_moves_urllib_robotparser, attr.name, attr) +del attr +Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes +_importer._add_module( + Module_six_moves_urllib_robotparser( + __name__ + ".moves.urllib.robotparser" + ), + "moves.urllib_robotparser", + "moves.urllib.robotparser", +) + + +class Module_six_moves_urllib(types.ModuleType): + """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + __path__ = [] # mark as package + parse = _importer._get_module("moves.urllib_parse") + error = _importer._get_module("moves.urllib_error") + request = _importer._get_module("moves.urllib_request") + response = _importer._get_module("moves.urllib_response") + robotparser = _importer._get_module("moves.urllib_robotparser") + + def __dir__(self): + return ['parse', 'error', 'request', 'response', 'robotparser'] + + +_importer._add_module( + Module_six_moves_urllib(__name__ + ".moves.urllib"), "moves.urllib" +) + + +def add_move(move): + """Add an item to six.moves.""" + setattr(_MovedItems, move.name, move) + + +def remove_move(name): + """Remove item from six.moves.""" + try: + delattr(_MovedItems, name) + except AttributeError: + try: + del moves.__dict__[name] + except KeyError: + raise AttributeError("no such move, %r" % (name,)) + + +if PY3: + _meth_func = "__func__" + _meth_self = "__self__" + _func_closure = "__closure__" + _func_code = "__code__" + _func_defaults = "__defaults__" + _func_globals = "__globals__" +else: + _meth_func = "im_func" + _meth_self = "im_self" + _func_closure = "func_closure" + _func_code = "func_code" + _func_defaults = "func_defaults" + _func_globals = "func_globals" +try: + advance_iterator = next +except NameError: + + def advance_iterator(it): + return it.next() + + +next = advance_iterator +try: + callable = callable +except NameError: + + def callable(obj): + return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) + + +if PY3: + + def get_unbound_function(unbound): + return unbound + + create_bound_method = types.MethodType + + def create_unbound_method(func, cls): + return func + + Iterator = object +else: + + def get_unbound_function(unbound): + return unbound.im_func + + def create_bound_method(func, obj): + return types.MethodType(func, obj, obj.__class__) + + def create_unbound_method(func, cls): + return types.MethodType(func, None, cls) + + class Iterator(object): + + def next(self): + return type(self).__next__(self) + + callable = callable +_add_doc( + get_unbound_function, + """Get the function out of a possibly unbound function""", +) +get_method_function = operator.attrgetter(_meth_func) +get_method_self = operator.attrgetter(_meth_self) +get_function_closure = operator.attrgetter(_func_closure) +get_function_code = operator.attrgetter(_func_code) +get_function_defaults = operator.attrgetter(_func_defaults) +get_function_globals = operator.attrgetter(_func_globals) +if PY3: + + def iterkeys(d, **kw): + return iter(d.keys(**kw)) + + def itervalues(d, **kw): + return iter(d.values(**kw)) + + def iteritems(d, **kw): + return iter(d.items(**kw)) + + def iterlists(d, **kw): + return iter(d.lists(**kw)) + + viewkeys = operator.methodcaller("keys") + viewvalues = operator.methodcaller("values") + viewitems = operator.methodcaller("items") +else: + + def iterkeys(d, **kw): + return d.iterkeys(**kw) + + def itervalues(d, **kw): + return d.itervalues(**kw) + + def iteritems(d, **kw): + return d.iteritems(**kw) + + def iterlists(d, **kw): + return d.iterlists(**kw) + + viewkeys = operator.methodcaller("viewkeys") + viewvalues = operator.methodcaller("viewvalues") + viewitems = operator.methodcaller("viewitems") +_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") +_add_doc(itervalues, "Return an iterator over the values of a dictionary.") +_add_doc( + iteritems, + "Return an iterator over the (key, value) pairs of a dictionary.", +) +_add_doc( + iterlists, + "Return an iterator over the (key, [values]) pairs of a dictionary.", +) +if PY3: + + def b(s): + return s.encode("latin-1") + + def u(s): + return s + + unichr = chr + import struct + + int2byte = struct.Struct(">B").pack + del struct + byte2int = operator.itemgetter(0) + indexbytes = operator.getitem + iterbytes = iter + import io + + StringIO = io.StringIO + BytesIO = io.BytesIO + _assertCountEqual = "assertCountEqual" + if sys.version_info[1] <= 1: + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" + else: + _assertRaisesRegex = "assertRaisesRegex" + _assertRegex = "assertRegex" +else: + + def b(s): + return s + + + # Workaround for standalone backslash + def u(s): + return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + + unichr = unichr + int2byte = chr + + def byte2int(bs): + return ord(bs[0]) + + def indexbytes(buf, i): + return ord(buf[i]) + + iterbytes = functools.partial(itertools.imap, ord) + import StringIO + + StringIO = BytesIO = StringIO.StringIO + _assertCountEqual = "assertItemsEqual" + _assertRaisesRegex = "assertRaisesRegexp" + _assertRegex = "assertRegexpMatches" +_add_doc(b, """Byte literal""") +_add_doc(u, """Text literal""") + + +def assertCountEqual(self, *args, **kwargs): + return getattr(self, _assertCountEqual)(*args, **kwargs) + + +def assertRaisesRegex(self, *args, **kwargs): + return getattr(self, _assertRaisesRegex)(*args, **kwargs) + + +def assertRegex(self, *args, **kwargs): + return getattr(self, _assertRegex)(*args, **kwargs) + + +if PY3: + exec_ = getattr(moves.builtins, "exec") + + def reraise(tp, value, tb=None): + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + + raise value + + +else: + + def exec_(_code_, _globs_=None, _locs_=None): + """Execute code in a namespace.""" + if _globs_ is None: + frame = sys._getframe(1) + _globs_ = frame.f_globals + if _locs_ is None: + _locs_ = frame.f_locals + del frame + elif _locs_ is None: + _locs_ = _globs_ + exec ("""exec _code_ in _globs_, _locs_""") + + exec_( + """def reraise(tp, value, tb=None): + raise tp, value, tb +""" + ) +if sys.version_info[:2] == (3, 2): + exec_( + """def raise_from(value, from_value): + if from_value is None: + raise value + raise value from from_value +""" + ) +elif sys.version_info[:2] > (3, 2): + exec_( + """def raise_from(value, from_value): + raise value from from_value +""" + ) +else: + + def raise_from(value, from_value): + raise value + + +print_ = getattr(moves.builtins, "print", None) +if print_ is None: + + def print_(*args, **kwargs): + """The new-style print function for Python 2.4 and 2.5.""" + fp = kwargs.pop("file", sys.stdout) + if fp is None: + return + + def write(data): + if not isinstance(data, basestring): + data = str(data) + # If the file has an encoding, encode unicode with it. + if ( + isinstance(fp, file) and + isinstance(data, unicode) and + fp.encoding is not None + ): + errors = getattr(fp, "errors", None) + if errors is None: + errors = "strict" + data = data.encode(fp.encoding, errors) + fp.write(data) + + want_unicode = False + sep = kwargs.pop("sep", None) + if sep is not None: + if isinstance(sep, unicode): + want_unicode = True + elif not isinstance(sep, str): + raise TypeError("sep must be None or a string") + + end = kwargs.pop("end", None) + if end is not None: + if isinstance(end, unicode): + want_unicode = True + elif not isinstance(end, str): + raise TypeError("end must be None or a string") + + if kwargs: + raise TypeError("invalid keyword arguments to print()") + + if not want_unicode: + for arg in args: + if isinstance(arg, unicode): + want_unicode = True + break + + if want_unicode: + newline = unicode("\n") + space = unicode(" ") + else: + newline = "\n" + space = " " + if sep is None: + sep = space + if end is None: + end = newline + for i, arg in enumerate(args): + if i: + write(sep) + write(arg) + write(end) + + +if sys.version_info[:2] < (3, 3): + _print = print_ + + def print_(*args, **kwargs): + fp = kwargs.get("file", sys.stdout) + flush = kwargs.pop("flush", False) + _print(*args, **kwargs) + if flush and fp is not None: + fp.flush() + + +_add_doc(reraise, """Reraise an exception.""") +if sys.version_info[0:2] < (3, 4): + + def wraps( + wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES, + ): + + def wrapper(f): + f = functools.wraps(wrapped, assigned, updated)(f) + f.__wrapped__ = wrapped + return f + + return wrapper + + +else: + wraps = functools.wraps + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + + # This requires a bit of explanation: the basic idea is to make a dummy + # metaclass for one level of class instantiation that replaces itself with + # the actual metaclass. + class metaclass(meta): + + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + + return type.__new__(metaclass, 'temporary_class', (), {}) + + +def add_metaclass(metaclass): + """Class decorator for creating a class with a metaclass.""" + + def wrapper(cls): + orig_vars = cls.__dict__.copy() + slots = orig_vars.get('__slots__') + if slots is not None: + if isinstance(slots, str): + slots = [slots] + for slots_var in slots: + orig_vars.pop(slots_var) + orig_vars.pop('__dict__', None) + orig_vars.pop('__weakref__', None) + return metaclass(cls.__name__, cls.__bases__, orig_vars) + + return wrapper + + +def python_2_unicode_compatible(klass): + """ + A decorator that defines __unicode__ and __str__ methods under Python 2. + Under Python 3 it does nothing. + + To support Python 2 and 3 with a single code base, define a __str__ method + returning text and apply this decorator to the class. + """ + if PY2: + if '__str__' not in klass.__dict__: + raise ValueError( + "@python_2_unicode_compatible cannot be applied " + "to %s because it doesn't define __str__()." % klass.__name__ + ) + + klass.__unicode__ = klass.__str__ + klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + return klass + + +# Complete the moves implementation. +# This code is at the end of this module to speed up module loading. +# Turn this module into a package. +__path__ = [] # required for PEP 302 and PEP 451 +__package__ = __name__ # see PEP 366 @ReservedAssignment +if globals().get("__spec__") is not None: + __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable +# Remove other six meta path importers, since they cause problems. This can +# happen if six is removed from sys.modules and then reloaded. (Setuptools does +# this for some reason.) +if sys.meta_path: + for i, importer in enumerate(sys.meta_path): + # Here's some real nastiness: Another "instance" of the six module might + # be floating around. Therefore, we can't use isinstance() to check for + # the six meta path importer, since the other six instance will have + # inserted an importer with different class. + if ( + type(importer).__name__ == "_SixMetaPathImporter" and + importer.name == __name__ + ): + del sys.meta_path[i] + break + + del i, importer +# Finally, add the importer to the meta path import hook. +sys.meta_path.append(_importer) diff --git a/requests3/core/_http/packages/ssl_match_hostname/__init__.py b/requests3/core/_http/packages/ssl_match_hostname/__init__.py new file mode 100644 index 00000000..612b8a0b --- /dev/null +++ b/requests3/core/_http/packages/ssl_match_hostname/__init__.py @@ -0,0 +1,18 @@ +import sys + +try: + # Our match_hostname function is the same as 3.5's, so we only want to + # import the match_hostname function if it's at least that good. + if sys.version_info < (3, 5): + raise ImportError("Fallback to vendored code") + + from ssl import CertificateError, match_hostname +except ImportError: + try: + # Backport of the function from a pypi module + from backports.ssl_match_hostname import CertificateError, match_hostname + except ImportError: + # Our vendored copy + from ._implementation import CertificateError, match_hostname +# Not needed, but documenting what we provide. +__all__ = ('CertificateError', 'match_hostname') diff --git a/requests3/core/_http/packages/ssl_match_hostname/_implementation.py b/requests3/core/_http/packages/ssl_match_hostname/_implementation.py new file mode 100644 index 00000000..925bad60 --- /dev/null +++ b/requests3/core/_http/packages/ssl_match_hostname/_implementation.py @@ -0,0 +1,165 @@ +"""The match_hostname() function from Python 3.3.3, essential when using SSL.""" +# Note: This file is under the PSF license as the code comes from the python +# stdlib. http://docs.python.org/3/license.html +import re +import sys + +# ipaddress has been backported to 2.6+ in pypi. If it is installed on the +# system, use it to handle IPAddress ServerAltnames (this was added in +# python-3.5) otherwise only do DNS matching. This allows +# backports.ssl_match_hostname to continue to be used all the way back to +# python-2.4. +try: + import ipaddress +except ImportError: + ipaddress = None +__version__ = '3.5.0.1' + + +class CertificateError(ValueError): + pass + + +def _dnsname_match(dn, hostname, max_wildcards=1): + """Matching according to RFC 6125, section 6.4.3 + + http://tools.ietf.org/html/rfc6125#section-6.4.3 + """ + pats = [] + if not dn: + return False + + # Ported from python3-syntax: + # leftmost, *remainder = dn.split(r'.') + parts = dn.split(r'.') + leftmost = parts[0] + remainder = parts[1:] + wildcards = leftmost.count('*') + if wildcards > max_wildcards: + # Issue #17980: avoid denials of service by refusing more + # than one wildcard per fragment. A survey of established + # policy among SSL implementations showed it to be a + # reasonable choice. + raise CertificateError( + "too many wildcards in certificate DNS name: " + repr(dn) + ) + + # speed up common case w/o wildcards + if not wildcards: + return dn.lower() == hostname.lower() + + # RFC 6125, section 6.4.3, subitem 1. + # The client SHOULD NOT attempt to match a presented identifier in which + # the wildcard character comprises a label other than the left-most label. + if leftmost == '*': + # When '*' is a fragment by itself, it matches a non-empty dotless + # fragment. + pats.append('[^.]+') + elif leftmost.startswith('xn--') or hostname.startswith('xn--'): + # RFC 6125, section 6.4.3, subitem 3. + # The client SHOULD NOT attempt to match a presented identifier + # where the wildcard character is embedded within an A-label or + # U-label of an internationalized domain name. + pats.append(re.escape(leftmost)) + else: + # Otherwise, '*' matches any dotless string, e.g. www* + pats.append(re.escape(leftmost).replace(r'\*', '[^.]*')) + # add the remaining fragments, ignore any wildcards + for frag in remainder: + pats.append(re.escape(frag)) + pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) + return pat.match(hostname) + + +def _to_unicode(obj): + if isinstance(obj, str) and sys.version_info < (3,): + obj = unicode(obj, encoding='ascii', errors='strict') + return obj + + +def _ipaddress_match(ipname, host_ip): + """Exact matching of IP addresses. + + RFC 6125 explicitly doesn't define an algorithm for this + (section 1.7.2 - "Out of Scope"). + """ + # OpenSSL may add a trailing newline to a subjectAltName's IP address + # Divergence from upstream: ipaddress can't handle byte str + ip = ipaddress.ip_address(_to_unicode(ipname).rstrip()) + return ip == host_ip + + +def match_hostname(cert, hostname): + """Verify that *cert* (in decoded format as returned by + SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 + rules are followed, but IP addresses are not accepted for *hostname*. + + CertificateError is raised on failure. On success, the function + returns nothing. + """ + if not cert: + raise ValueError( + "empty or no certificate, match_hostname needs a " + "SSL socket or SSL context with either " + "CERT_OPTIONAL or CERT_REQUIRED" + ) + + try: + # Divergence from upstream: ipaddress can't handle byte str + host_ip = ipaddress.ip_address(_to_unicode(hostname)) + except ValueError: + # Not an IP address (common case) + host_ip = None + except UnicodeError: + # Divergence from upstream: Have to deal with ipaddress not taking + # byte strings. addresses should be all ascii, so we consider it not + # an ipaddress in this case + host_ip = None + except AttributeError: + # Divergence from upstream: Make ipaddress library optional + if ipaddress is None: + host_ip = None + else: + raise + + dnsnames = [] + san = cert.get('subjectAltName', ()) + for key, value in san: + if key == 'DNS': + if host_ip is None and _dnsname_match(value, hostname): + return + + dnsnames.append(value) + elif key == 'IP Address': + if host_ip is not None and _ipaddress_match(value, host_ip): + return + + dnsnames.append(value) + if not dnsnames: + # The subject is only checked when there is no dNSName entry + # in subjectAltName + for sub in cert.get('subject', ()): + for key, value in sub: + # XXX according to RFC 2818, the most specific Common Name + # must be used. + if key == 'commonName': + if _dnsname_match(value, hostname): + return + + dnsnames.append(value) + if len(dnsnames) > 1: + raise CertificateError( + "hostname %r " + "doesn't match either of %s" % + (hostname, ', '.join(map(repr, dnsnames))) + ) + + elif len(dnsnames) == 1: + raise CertificateError( + "hostname %r " "doesn't match %r" % (hostname, dnsnames[0]) + ) + + else: + raise CertificateError( + "no appropriate commonName or " "subjectAltName fields were found" + ) diff --git a/requests3/core/_http/poolmanager.py b/requests3/core/_http/poolmanager.py new file mode 100644 index 00000000..62bf7dd8 --- /dev/null +++ b/requests3/core/_http/poolmanager.py @@ -0,0 +1,3 @@ +from ._sync.poolmanager import PoolManager, ProxyManager, proxy_from_url + +__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] diff --git a/requests3/core/_http/request.py b/requests3/core/_http/request.py new file mode 100644 index 00000000..bc2e0cbb --- /dev/null +++ b/requests3/core/_http/request.py @@ -0,0 +1,163 @@ +from __future__ import absolute_import + +from .filepost import encode_multipart_formdata +from .packages import six +from .packages.six.moves.urllib.parse import urlencode + +__all__ = ['RequestMethods'] + + +class RequestMethods(object): + """ + Convenience mixin for classes who implement a :meth:`urlopen` method, such + as :class:`~urllib3.connectionpool.HTTPConnectionPool` and + :class:`~urllib3.poolmanager.PoolManager`. + + Provides behavior for making common types of HTTP request methods and + decides which type of request field encoding to use. + + Specifically, + + :meth:`.request_encode_url` is for sending requests whose fields are + encoded in the URL (such as GET, HEAD, DELETE). + + :meth:`.request_encode_body` is for sending requests whose fields are + encoded in the *body* of the request using multipart or www-form-urlencoded + (such as for POST, PUT, PATCH). + + :meth:`.request` is for making any kind of request, it will look up the + appropriate encoding format and use one of the above two methods to make + the request. + + Initializer parameters: + + :param headers: + Headers to include with all requests, unless other headers are given + explicitly. + """ + _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS']) + + def __init__(self, headers=None): + self.headers = headers or {} + + def urlopen( + self, + method, + url, + body=None, + headers=None, + encode_multipart=True, + multipart_boundary=None, + **kw + ): # Abstract + raise NotImplementedError( + "Classes extending RequestMethods must implement " + "their own ``urlopen`` method." + ) + + def request(self, method, url, fields=None, headers=None, **urlopen_kw): + """ + Make a request using :meth:`urlopen` with the appropriate encoding of + ``fields`` based on the ``method`` used. + + This is a convenience method that requires the least amount of manual + effort. It can be used in most situations, while still having the + option to drop down to more specific methods when necessary, such as + :meth:`request_encode_url`, :meth:`request_encode_body`, + or even the lowest level :meth:`urlopen`. + """ + method = method.upper() + if method in self._encode_url_methods: + return self.request_encode_url( + method, url, fields=fields, headers=headers, **urlopen_kw + ) + + else: + return self.request_encode_body( + method, url, fields=fields, headers=headers, **urlopen_kw + ) + + def request_encode_url( + self, method, url, fields=None, headers=None, **urlopen_kw + ): + """ + Make a request using :meth:`urlopen` with the ``fields`` encoded in + the url. This is useful for request methods like GET, HEAD, DELETE, etc. + """ + if headers is None: + headers = self.headers + extra_kw = {'headers': headers} + extra_kw.update(urlopen_kw) + if fields: + url += '?' + urlencode(fields) + return self.urlopen(method, url, **extra_kw) + + def request_encode_body( + self, + method, + url, + fields=None, + headers=None, + encode_multipart=True, + multipart_boundary=None, + **urlopen_kw + ): + """ + Make a request using :meth:`urlopen` with the ``fields`` encoded in + the body. This is useful for request methods like POST, PUT, PATCH, etc. + + When ``encode_multipart=True`` (default), then + :meth:`urllib3.filepost.encode_multipart_formdata` is used to encode + the payload with the appropriate content type. Otherwise + :meth:`urllib.urlencode` is used with the + 'application/x-www-form-urlencoded' content type. + + Multipart encoding must be used when posting files, and it's reasonably + safe to use it in other times too. However, it may break request + signing, such as with OAuth. + + Supports an optional ``fields`` parameter of key/value strings AND + key/filetuple. A filetuple is a (filename, data, MIME type) tuple where + the MIME type is optional. For example:: + + fields = { + 'foo': 'bar', + 'fakefile': ('foofile.txt', 'contents of foofile'), + 'realfile': ('barfile.txt', open('realfile').read()), + 'typedfile': ('bazfile.bin', open('bazfile').read(), + 'image/jpeg'), + 'nonamefile': 'contents of nonamefile field', + } + + When uploading a file, providing a filename (the first parameter of the + tuple) is optional but recommended to best mimick behavior of browsers. + + Note that if ``headers`` are supplied, the 'Content-Type' header will + be overwritten because it depends on the dynamic random boundary string + which is used to compose the body of the request. The random boundary + string can be explicitly set with the ``multipart_boundary`` parameter. + """ + if headers is None: + headers = self.headers + extra_kw = {'headers': {}} + if fields: + if 'body' in urlopen_kw: + raise TypeError( + "request got values for both 'fields' and 'body', can only specify one." + ) + + if encode_multipart: + body, content_type = encode_multipart_formdata( + fields, boundary=multipart_boundary + ) + else: + body, content_type = urlencode( + fields + ), 'application/x-www-form-urlencoded' + if isinstance(body, six.text_type): + body = body.encode('utf-8') + extra_kw['body'] = body + extra_kw['headers'] = {'Content-Type': content_type} + extra_kw['headers'].update(headers) + extra_kw.update(urlopen_kw) + return self.urlopen(method, url, **extra_kw) diff --git a/requests3/core/_http/response.py b/requests3/core/_http/response.py new file mode 100644 index 00000000..1e95d13d --- /dev/null +++ b/requests3/core/_http/response.py @@ -0,0 +1,3 @@ +from ._sync.response import DeflateDecoder, GzipDecoder, HTTPResponse + +__all__ = ['DeflateDecoder', 'GzipDecoder', 'HTTPResponse'] diff --git a/requests3/core/_http/util/__init__.py b/requests3/core/_http/util/__init__.py new file mode 100644 index 00000000..9014131b --- /dev/null +++ b/requests3/core/_http/util/__init__.py @@ -0,0 +1,44 @@ +from __future__ import absolute_import + +# For backwards compatibility, provide imports that used to be here. +from .connection import is_connection_dropped +from .request import make_headers +from .response import is_fp_closed +from .ssl_ import ( + SSLContext, + HAS_SNI, + IS_PYOPENSSL, + IS_SECURETRANSPORT, + assert_fingerprint, + resolve_cert_reqs, + resolve_ssl_version, + ssl_wrap_socket, +) +from .timeout import (current_time, Timeout) + +from .retry import Retry +from .url import (get_host, parse_url, split_first, Url) +from .wait import (wait_for_read, wait_for_write) + +__all__ = ( + 'HAS_SNI', + 'IS_PYOPENSSL', + 'IS_SECURETRANSPORT', + 'SSLContext', + 'Retry', + 'Timeout', + 'Url', + 'assert_fingerprint', + 'current_time', + 'is_connection_dropped', + 'is_fp_closed', + 'get_host', + 'parse_url', + 'make_headers', + 'resolve_cert_reqs', + 'resolve_ssl_version', + 'split_first', + 'ssl_wrap_socket', + 'wait_for_read', + 'wait_for_write', +) diff --git a/requests3/core/_http/util/connection.py b/requests3/core/_http/util/connection.py new file mode 100644 index 00000000..89a1ca32 --- /dev/null +++ b/requests3/core/_http/util/connection.py @@ -0,0 +1,108 @@ +from __future__ import absolute_import +import socket + + +def is_connection_dropped(conn): # Platform-specific + """ + Returns True if the connection is dropped and should be closed. + """ + # TODO: Need to restore AppEngine behaviour here at some point. + return conn.is_dropped() + + + + +# This function is copied from socket.py in the Python 2.7 standard +# library test suite. Added to its signature is only `socket_options`. +# One additional modification is that we avoid binding to IPv6 servers +# discovered in DNS if the system doesn't have IPv6 functionality. +def create_connection( + address, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + source_address=None, + socket_options=None, +): + """Connect to *address* and return the socket object. + + Convenience function. Connect to *address* (a 2-tuple ``(host, + port)``) and return the socket object. Passing the optional + *timeout* parameter will set the timeout on the socket instance + before attempting to connect. If no *timeout* is supplied, the + global default timeout setting returned by :func:`getdefaulttimeout` + is used. If *source_address* is set it must be a tuple of (host, port) + for the socket to bind as a source address before making the connection. + An host of '' or port 0 tells the OS to use the default. + """ + host, port = address + if host.startswith('['): + host = host.strip('[]') + err = None + # Using the value from allowed_gai_family() in the context of getaddrinfo lets + # us select whether to work with IPv4 DNS records, IPv6 records, or both. + # The original create_connection function always returns all records. + family = allowed_gai_family() + for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + sock = None + try: + sock = socket.socket(af, socktype, proto) + # If provided, set socket level options before connecting. + _set_socket_options(sock, socket_options) + if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: + sock.settimeout(timeout) + if source_address: + sock.bind(source_address) + sock.connect(sa) + return sock + + except socket.error as e: + err = e + if sock is not None: + sock.close() + sock = None + if err is not None: + raise err + + raise socket.error("getaddrinfo returns an empty list") + + +def _set_socket_options(sock, options): + if options is None: + return + + for opt in options: + sock.setsockopt(*opt) + + +def allowed_gai_family(): + """This function is designed to work in the context of + getaddrinfo, where family=socket.AF_UNSPEC is the default and + will perform a DNS search for both IPv6 and IPv4 records.""" + family = socket.AF_INET + if HAS_IPV6: + family = socket.AF_UNSPEC + return family + + +def _has_ipv6(host): + """ Returns True if the system can bind an IPv6 address. """ + sock = None + has_ipv6 = False + if socket.has_ipv6: + # has_ipv6 returns true if cPython was compiled with IPv6 support. + # It does not tell us if the system has IPv6 support enabled. To + # determine that we must bind to an IPv6 address. + # https://github.com/shazow/urllib3/pull/611 + # https://bugs.python.org/issue658327 + try: + sock = socket.socket(socket.AF_INET6) + sock.bind((host, 0)) + has_ipv6 = True + except Exception: + pass + if sock: + sock.close() + return has_ipv6 + + +HAS_IPV6 = _has_ipv6('::1') diff --git a/requests3/core/_http/util/request.py b/requests3/core/_http/util/request.py new file mode 100644 index 00000000..43102ff1 --- /dev/null +++ b/requests3/core/_http/util/request.py @@ -0,0 +1,129 @@ +from __future__ import absolute_import +from base64 import b64encode + +from ..packages.six import b, integer_types +from ..exceptions import UnrewindableBodyError + +ACCEPT_ENCODING = 'gzip,deflate' +_FAILEDTELL = object() + + +def make_headers( + keep_alive=None, + accept_encoding=None, + user_agent=None, + basic_auth=None, + proxy_basic_auth=None, + disable_cache=None, +): + """ + Shortcuts for generating request headers. + + :param keep_alive: + If ``True``, adds 'connection: keep-alive' header. + + :param accept_encoding: + Can be a boolean, list, or string. + ``True`` translates to 'gzip,deflate'. + List will get joined by comma. + String will be used as provided. + + :param user_agent: + String representing the user-agent you want, such as + "python-urllib3/0.6" + + :param basic_auth: + Colon-separated username:password string for 'authorization: basic ...' + auth header. + + :param proxy_basic_auth: + Colon-separated username:password string for 'proxy-authorization: basic ...' + auth header. + + :param disable_cache: + If ``True``, adds 'cache-control: no-cache' header. + + Example:: + + >>> make_headers(keep_alive=True, user_agent="Batman/1.0") + {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} + >>> make_headers(accept_encoding=True) + {'accept-encoding': 'gzip,deflate'} + """ + headers = {} + if accept_encoding: + if isinstance(accept_encoding, str): + pass + elif isinstance(accept_encoding, list): + accept_encoding = ','.join(accept_encoding) + else: + accept_encoding = ACCEPT_ENCODING + headers['accept-encoding'] = accept_encoding + if user_agent: + headers['user-agent'] = user_agent + if keep_alive: + headers['connection'] = 'keep-alive' + if basic_auth: + headers['authorization'] = 'Basic ' + b64encode(b(basic_auth)).decode( + 'utf-8' + ) + if proxy_basic_auth: + headers['proxy-authorization'] = 'Basic ' + b64encode( + b(proxy_basic_auth) + ).decode( + 'utf-8' + ) + if disable_cache: + headers['cache-control'] = 'no-cache' + return headers + + +def set_file_position(body, pos): + """ + If a position is provided, move file to that point. + Otherwise, we'll attempt to record a position for future use. + """ + if pos is not None: + rewind_body(body, pos) + elif getattr(body, 'tell', None) is not None: + try: + pos = body.tell() + except (IOError, OSError): + # This differentiates from None, allowing us to catch + # a failed `tell()` later when trying to rewind the body. + pos = _FAILEDTELL + return pos + + +def rewind_body(body, body_pos): + """ + Attempt to rewind body to a certain position. + Primarily used for request redirects and retries. + + :param body: + File-like object that supports seek. + + :param int pos: + Position to seek to in file. + """ + body_seek = getattr(body, 'seek', None) + if body_seek is not None and isinstance(body_pos, integer_types): + try: + body_seek(body_pos) + except (IOError, OSError): + raise UnrewindableBodyError( + "An error occurred when rewinding request " + "body for redirect/retry." + ) + + elif body_pos is _FAILEDTELL: + raise UnrewindableBodyError( + "Unable to record file position for rewinding " + "request body during a redirect/retry." + ) + + else: + raise ValueError( + "body_pos must be of type integer, " + "instead it was %s." % type(body_pos) + ) diff --git a/requests3/core/_http/util/response.py b/requests3/core/_http/util/response.py new file mode 100644 index 00000000..4f31a85f --- /dev/null +++ b/requests3/core/_http/util/response.py @@ -0,0 +1,30 @@ +from __future__ import absolute_import + + +def is_fp_closed(obj): + """ + Checks whether a given file-like object is closed. + + :param obj: + The file-like object to check. + """ + try: + # Check for our own base response class. + return obj.complete + + except AttributeError: + pass + try: + # Check via the official file-like-object way. + return obj.closed + + except AttributeError: + pass + try: + # Check if the object is a container for another file-like object that + # gets released on exhaustion (e.g. HTTPResponse). + return obj.fp is None + + except AttributeError: + pass + raise ValueError("Unable to determine whether fp is closed.") diff --git a/requests3/core/_http/util/retry.py b/requests3/core/_http/util/retry.py new file mode 100644 index 00000000..01157f95 --- /dev/null +++ b/requests3/core/_http/util/retry.py @@ -0,0 +1,432 @@ +from __future__ import absolute_import +import time +import logging +from collections import namedtuple +from itertools import takewhile +import email +import re + +from ..exceptions import ( + ConnectTimeoutError, + MaxRetryError, + ProtocolError, + ReadTimeoutError, + ResponseError, + InvalidHeader, +) +from ..packages import six + +log = logging.getLogger(__name__) +# Data structure for representing the metadata of requests that result in a retry. +RequestHistory = namedtuple( + 'RequestHistory', ["method", "url", "error", "status", "redirect_location"] +) + + +class Retry(object): + """ Retry configuration. + + Each retry attempt will create a new Retry object with updated values, so + they can be safely reused. + + Retries can be defined as a default for a pool:: + + retries = Retry(connect=5, read=2, redirect=5) + http = PoolManager(retries=retries) + response = http.request('GET', 'http://example.com/') + + Or per-request (which overrides the default for the pool):: + + response = http.request('GET', 'http://example.com/', retries=Retry(10)) + + Retries can be disabled by passing ``False``:: + + response = http.request('GET', 'http://example.com/', retries=False) + + Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless + retries are disabled, in which case the causing exception will be raised. + + :param int total: + Total number of retries to allow. Takes precedence over other counts. + + Set to ``None`` to remove this constraint and fall back on other + counts. It's a good idea to set this to some sensibly-high value to + account for unexpected edge cases and avoid infinite retry loops. + + Set to ``0`` to fail on the first retry. + + Set to ``False`` to disable and imply ``raise_on_redirect=False``. + + :param int connect: + How many connection-related errors to retry on. + + These are errors raised before the request is sent to the remote server, + which we assume has not triggered the server to process the request. + + Set to ``0`` to fail on the first retry of this type. + + :param int read: + How many times to retry on read errors. + + These errors are raised after the request was sent to the server, so the + request may have side-effects. + + Set to ``0`` to fail on the first retry of this type. + + :param int redirect: + How many redirects to perform. Limit this to avoid infinite redirect + loops. + + A redirect is a HTTP response with a status code 301, 302, 303, 307 or + 308. + + Set to ``0`` to fail on the first retry of this type. + + Set to ``False`` to disable and imply ``raise_on_redirect=False``. + + :param int status: + How many times to retry on bad status codes. + + These are retries made on responses, where status code matches + ``status_forcelist``. + + Set to ``0`` to fail on the first retry of this type. + + :param iterable method_whitelist: + Set of uppercased HTTP method verbs that we should retry on. + + By default, we only retry on methods which are considered to be + idempotent (multiple requests with the same parameters end with the + same state). See :attr:`Retry.DEFAULT_METHOD_WHITELIST`. + + Set to a ``False`` value to retry on any verb. + + :param iterable status_forcelist: + A set of integer HTTP status codes that we should force a retry on. + A retry is initiated if the request method is in ``method_whitelist`` + and the response status code is in ``status_forcelist``. + + By default, this is disabled with ``None``. + + :param float backoff_factor: + A backoff factor to apply between attempts after the second try + (most errors are resolved immediately by a second try without a + delay). urllib3 will sleep for:: + + {backoff factor} * (2 ^ ({number of total retries} - 1)) + + seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep + for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer + than :attr:`Retry.BACKOFF_MAX`. + + By default, backoff is disabled (set to 0). + + :param bool raise_on_redirect: Whether, if the number of redirects is + exhausted, to raise a MaxRetryError, or to return a response with a + response code in the 3xx range. + + :param bool raise_on_status: Similar meaning to ``raise_on_redirect``: + whether we should raise an exception, or return a response, + if status falls in ``status_forcelist`` range and retries have + been exhausted. + + :param tuple history: The history of the request encountered during + each call to :meth:`~Retry.increment`. The list is in the order + the requests occurred. Each list item is of class :class:`RequestHistory`. + + :param bool respect_retry_after_header: + Whether to respect Retry-After header on status codes defined as + :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not. + + """ + DEFAULT_METHOD_WHITELIST = frozenset( + ['HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE'] + ) + RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) + # : Maximum backoff time. + BACKOFF_MAX = 120 + + def __init__( + self, + total=10, + connect=None, + read=None, + redirect=None, + status=None, + method_whitelist=DEFAULT_METHOD_WHITELIST, + status_forcelist=None, + backoff_factor=0, + raise_on_redirect=True, + raise_on_status=True, + history=None, + respect_retry_after_header=True, + ): + self.total = total + self.connect = connect + self.read = read + self.status = status + if redirect is False or total is False: + redirect = 0 + raise_on_redirect = False + self.redirect = redirect + self.status_forcelist = status_forcelist or set() + self.method_whitelist = method_whitelist + self.backoff_factor = backoff_factor + self.raise_on_redirect = raise_on_redirect + self.raise_on_status = raise_on_status + self.history = history or tuple() + self.respect_retry_after_header = respect_retry_after_header + + def new(self, **kw): + params = dict( + total=self.total, + connect=self.connect, + read=self.read, + redirect=self.redirect, + status=self.status, + method_whitelist=self.method_whitelist, + status_forcelist=self.status_forcelist, + backoff_factor=self.backoff_factor, + raise_on_redirect=self.raise_on_redirect, + raise_on_status=self.raise_on_status, + history=self.history, + ) + params.update(kw) + return type(self)(**params) + + @classmethod + def from_int(cls, retries, redirect=True, default=None): + """ Backwards-compatibility for the old retries format.""" + if retries is None: + retries = default if default is not None else cls.DEFAULT + if isinstance(retries, Retry): + return retries + + redirect = bool(redirect) and None + new_retries = cls(retries, redirect=redirect) + log.debug("Converted retries value: %r -> %r", retries, new_retries) + return new_retries + + def get_backoff_time(self): + """ Formula for computing the current backoff + + :rtype: float + """ + # We want to consider only the last consecutive errors sequence (Ignore redirects). + consecutive_errors_len = len( + list( + takewhile( + lambda x: x.redirect_location is None, + reversed(self.history), + ) + ) + ) + if consecutive_errors_len <= 1: + return 0 + + backoff_value = self.backoff_factor * ( + 2 ** (consecutive_errors_len - 1) + ) + return min(self.BACKOFF_MAX, backoff_value) + + def parse_retry_after(self, retry_after): + # Whitespace: https://tools.ietf.org/html/rfc7230#section-3.2.4 + if re.match(r"^\s*[0-9]+\s*$", retry_after): + seconds = int(retry_after) + else: + retry_date_tuple = email.utils.parsedate(retry_after) + if retry_date_tuple is None: + raise InvalidHeader( + "Invalid Retry-After header: %s" % retry_after + ) + + retry_date = time.mktime(retry_date_tuple) + seconds = retry_date - time.time() + if seconds < 0: + seconds = 0 + return seconds + + def get_retry_after(self, response): + """ Get the value of Retry-After in seconds. """ + retry_after = response.getheader("Retry-After") + if retry_after is None: + return None + + return self.parse_retry_after(retry_after) + + def sleep_for_retry(self, response=None): + retry_after = self.get_retry_after(response) + if retry_after: + time.sleep(retry_after) + return True + + return False + + def _sleep_backoff(self): + backoff = self.get_backoff_time() + if backoff <= 0: + return + + time.sleep(backoff) + + def sleep(self, response=None): + """ Sleep between retry attempts. + + This method will respect a server's ``Retry-After`` response header + and sleep the duration of the time requested. If that is not present, it + will use an exponential backoff. By default, the backoff factor is 0 and + this method will return immediately. + """ + if response: + slept = self.sleep_for_retry(response) + if slept: + return + + self._sleep_backoff() + + def _is_connection_error(self, err): + """ Errors when we're fairly sure that the server did not receive the + request, so it should be safe to retry. + """ + return isinstance(err, ConnectTimeoutError) + + def _is_read_error(self, err): + """ Errors that occur after the request has been started, so we should + assume that the server began processing it. + """ + return isinstance(err, (ReadTimeoutError, ProtocolError)) + + def _is_method_retryable(self, method): + """ Checks if a given HTTP method should be retried upon, depending if + it is included on the method whitelist. + """ + if self.method_whitelist and method.upper( + ) not in self.method_whitelist: + return False + + return True + + def is_retry(self, method, status_code, has_retry_after=False): + """ Is this method/status code retryable? (Based on whitelists and control + variables such as the number of total retries to allow, whether to + respect the Retry-After header, whether this header is present, and + whether the returned status code is on the list of status codes to + be retried upon on the presence of the aforementioned header) + """ + if not self._is_method_retryable(method): + return False + + if self.status_forcelist and status_code in self.status_forcelist: + return True + + return ( + self.total and + self.respect_retry_after_header and + has_retry_after and + (status_code in self.RETRY_AFTER_STATUS_CODES) + ) + + def is_exhausted(self): + """ Are we out of retries? """ + retry_counts = ( + self.total, self.connect, self.read, self.redirect, self.status + ) + retry_counts = list(filter(None, retry_counts)) + if not retry_counts: + return False + + return min(retry_counts) < 0 + + def increment( + self, + method=None, + url=None, + response=None, + error=None, + _pool=None, + _stacktrace=None, + ): + """ Return a new Retry object with incremented retry counters. + + :param response: A response object, or None, if the server did not + return a response. + :type response: :class:`~urllib3.response.HTTPResponse` + :param Exception error: An error encountered during the request, or + None if the response was received successfully. + + :return: A new ``Retry`` object. + """ + if self.total is False and error: + # Disabled, indicate to re-raise the error. + raise six.reraise(type(error), error, _stacktrace) + + total = self.total + if total is not None: + total -= 1 + connect = self.connect + read = self.read + redirect = self.redirect + status_count = self.status + cause = 'unknown' + status = None + redirect_location = None + if error and self._is_connection_error(error): + # Connect retry? + if connect is False: + raise six.reraise(type(error), error, _stacktrace) + + elif connect is not None: + connect -= 1 + elif error and self._is_read_error(error): + # Read retry? + if read is False or not self._is_method_retryable(method): + raise six.reraise(type(error), error, _stacktrace) + + elif read is not None: + read -= 1 + elif response and response.get_redirect_location(): + # Redirect retry? + if redirect is not None: + redirect -= 1 + cause = 'too many redirects' + redirect_location = response.get_redirect_location() + status = response.status + else: + # Incrementing because of a server error like a 500 in + # status_forcelist and a the given method is in the whitelist + cause = ResponseError.GENERIC_ERROR + if response and response.status: + if status_count is not None: + status_count -= 1 + cause = ResponseError.SPECIFIC_ERROR.format( + status_code=response.status + ) + status = response.status + history = self.history + ( + RequestHistory(method, url, error, status, redirect_location), + ) + new_retry = self.new( + total=total, + connect=connect, + read=read, + redirect=redirect, + status=status_count, + history=history, + ) + if new_retry.is_exhausted(): + raise MaxRetryError(_pool, url, error or ResponseError(cause)) + + log.debug("Incremented Retry for (url='%s'): %r", url, new_retry) + return new_retry + + def __repr__(self): + return ( + '{cls.__name__}(total={self.total}, connect={self.connect}, ' + 'read={self.read}, redirect={self.redirect}, status={self.status})' + ).format( + cls=type(self), self=self + ) + + +# For backwards compatibility (equivalent to pre-v1.9): +Retry.DEFAULT = Retry(3) diff --git a/requests3/core/_http/util/selectors.py b/requests3/core/_http/util/selectors.py new file mode 100644 index 00000000..505f8082 --- /dev/null +++ b/requests3/core/_http/util/selectors.py @@ -0,0 +1,604 @@ +# Backport of selectors.py from Python 3.5+ to support Python < 3.4 +# Also has the behavior specified in PEP 475 which is to retry syscalls +# in the case of an EINTR error. This module is required because selectors34 +# does not follow this behavior and instead returns that no dile descriptor +# events have occurred rather than retry the syscall. The decision to drop +# support for select.devpoll is made to maintain 100% test coverage. +import errno +import math +import select +import socket +import sys +import time +from collections import namedtuple +from ..packages.six import integer_types + +try: + from collections.abc import Mapping +except ImportError: + from collections import Mapping +try: + monotonic = time.monotonic +except (AttributeError, ImportError): # Python 3.3< + monotonic = time.time +EVENT_READ = (1 << 0) +EVENT_WRITE = (1 << 1) +HAS_SELECT = True # Variable that shows whether the platform has a selector. +_SYSCALL_SENTINEL = object() # Sentinel in case a system call returns None. +_DEFAULT_SELECTOR = None + + +class SelectorError(Exception): + + def __init__(self, errcode): + super(SelectorError, self).__init__() + self.errno = errcode + + def __repr__(self): + return "".format(self.errno) + + def __str__(self): + return self.__repr__() + + +def _fileobj_to_fd(fileobj): + """ Return a file descriptor from a file object. If + given an integer will simply return that integer back. """ + if isinstance(fileobj, integer_types): + fd = fileobj + else: + try: + fd = int(fileobj.fileno()) + except (AttributeError, TypeError, ValueError): + raise ValueError("Invalid file object: {0!r}".format(fileobj)) + + if fd < 0: + raise ValueError("Invalid file descriptor: {0}".format(fd)) + + return fd + + +# Determine which function to use to wrap system calls because Python 3.5+ +# already handles the case when system calls are interrupted. +if sys.version_info >= (3, 5): + + def _syscall_wrapper(func, _, *args, **kwargs): + """ This is the short-circuit version of the below logic + because in Python 3.5+ all system calls automatically restart + and recalculate their timeouts. """ + try: + return func(*args, **kwargs) + + except (OSError, IOError, select.error) as e: + errcode = None + if hasattr(e, "errno"): + errcode = e.errno + raise SelectorError(errcode) + + +else: + + def _syscall_wrapper(func, recalc_timeout, *args, **kwargs): + """ Wrapper function for syscalls that could fail due to EINTR. + All functions should be retried if there is time left in the timeout + in accordance with PEP 475. """ + timeout = kwargs.get("timeout", None) + if timeout is None: + expires = None + recalc_timeout = False + else: + timeout = float(timeout) + if timeout < 0.0: # Timeout less than 0 treated as no timeout. + expires = None + else: + expires = monotonic() + timeout + args = list(args) + if recalc_timeout and "timeout" not in kwargs: + raise ValueError( + "Timeout must be in args or kwargs to be recalculated" + ) + + result = _SYSCALL_SENTINEL + while result is _SYSCALL_SENTINEL: + try: + result = func(*args, **kwargs) + # OSError is thrown by select.select + # IOError is thrown by select.epoll.poll + # select.error is thrown by select.poll.poll + # Aren't we thankful for Python 3.x rework for exceptions? + except (OSError, IOError, select.error) as e: + # select.error wasn't a subclass of OSError in the past. + errcode = None + if hasattr(e, "errno"): + errcode = e.errno + elif hasattr(e, "args"): + errcode = e.args[0] + # Also test for the Windows equivalent of EINTR. + is_interrupt = ( + errcode == errno.EINTR or + (hasattr(errno, "WSAEINTR") and errcode == errno.WSAEINTR) + ) + if is_interrupt: + if expires is not None: + current_time = monotonic() + if current_time > expires: + raise OSError(errno=errno.ETIMEDOUT) + + if recalc_timeout: + if "timeout" in kwargs: + kwargs["timeout"] = expires - current_time + continue + + if errcode: + raise SelectorError(errcode) + + else: + raise + + return result + + +SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data']) + + +class _SelectorMapping(Mapping): + """ Mapping of file objects to selector keys """ + + def __init__(self, selector): + self._selector = selector + + def __len__(self): + return len(self._selector._fd_to_key) + + def __getitem__(self, fileobj): + try: + fd = self._selector._fileobj_lookup(fileobj) + return self._selector._fd_to_key[fd] + + except KeyError: + raise KeyError("{0!r} is not registered.".format(fileobj)) + + def __iter__(self): + return iter(self._selector._fd_to_key) + + +class BaseSelector(object): + """ Abstract Selector class + + A selector supports registering file objects to be monitored + for specific I/O events. + + A file object is a file descriptor or any object with a + `fileno()` method. An arbitrary object can be attached to the + file object which can be used for example to store context info, + a callback, etc. + + A selector can use various implementations (select(), poll(), epoll(), + and kqueue()) depending on the platform. The 'DefaultSelector' class uses + the most efficient implementation for the current platform. + """ + + def __init__(self): + # Maps file descriptors to keys. + self._fd_to_key = {} + # Read-only mapping returned by get_map() + self._map = _SelectorMapping(self) + + def _fileobj_lookup(self, fileobj): + """ Return a file descriptor from a file object. + This wraps _fileobj_to_fd() to do an exhaustive + search in case the object is invalid but we still + have it in our map. Used by unregister() so we can + unregister an object that was previously registered + even if it is closed. It is also used by _SelectorMapping + """ + try: + return _fileobj_to_fd(fileobj) + + except ValueError: + # Search through all our mapped keys. + for key in self._fd_to_key.values(): + if key.fileobj is fileobj: + return key.fd + + # Raise ValueError after all. + raise + + def register(self, fileobj, events, data=None): + """ Register a file object for a set of events to monitor. """ + if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)): + raise ValueError("Invalid events: {0!r}".format(events)) + + key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data) + if key.fd in self._fd_to_key: + raise KeyError( + "{0!r} (FD {1}) is already registered".format(fileobj, key.fd) + ) + + self._fd_to_key[key.fd] = key + return key + + def unregister(self, fileobj): + """ Unregister a file object from being monitored. """ + try: + key = self._fd_to_key.pop(self._fileobj_lookup(fileobj)) + except KeyError: + raise KeyError("{0!r} is not registered".format(fileobj)) + + # Getting the fileno of a closed socket on Windows errors with EBADF. + except socket.error as e: # Platform-specific: Windows. + if e.errno != errno.EBADF: + raise + + else: + for key in self._fd_to_key.values(): + if key.fileobj is fileobj: + self._fd_to_key.pop(key.fd) + break + + else: + raise KeyError("{0!r} is not registered".format(fileobj)) + + return key + + def modify(self, fileobj, events, data=None): + """ Change a registered file object monitored events and data. """ + # NOTE: Some subclasses optimize this operation even further. + try: + key = self._fd_to_key[self._fileobj_lookup(fileobj)] + except KeyError: + raise KeyError("{0!r} is not registered".format(fileobj)) + + if events != key.events: + self.unregister(fileobj) + key = self.register(fileobj, events, data) + elif data != key.data: + # Use a shortcut to update the data. + key = key._replace(data=data) + self._fd_to_key[key.fd] = key + return key + + def select(self, timeout=None): + """ Perform the actual selection until some monitored file objects + are ready or the timeout expires. """ + raise NotImplementedError() + + def close(self): + """ Close the selector. This must be called to ensure that all + underlying resources are freed. """ + self._fd_to_key.clear() + self._map = None + + def get_key(self, fileobj): + """ Return the key associated with a registered file object. """ + mapping = self.get_map() + if mapping is None: + raise RuntimeError("Selector is closed") + + try: + return mapping[fileobj] + + except KeyError: + raise KeyError("{0!r} is not registered".format(fileobj)) + + def get_map(self): + """ Return a mapping of file objects to selector keys """ + return self._map + + def _key_from_fd(self, fd): + """ Return the key associated to a given file descriptor + Return None if it is not found. """ + try: + return self._fd_to_key[fd] + + except KeyError: + return None + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +# Almost all platforms have select.select() +if hasattr(select, "select"): + + class SelectSelector(BaseSelector): + """ Select-based selector. """ + + def __init__(self): + super(SelectSelector, self).__init__() + self._readers = set() + self._writers = set() + + def register(self, fileobj, events, data=None): + key = super(SelectSelector, self).register(fileobj, events, data) + if events & EVENT_READ: + self._readers.add(key.fd) + if events & EVENT_WRITE: + self._writers.add(key.fd) + return key + + def unregister(self, fileobj): + key = super(SelectSelector, self).unregister(fileobj) + self._readers.discard(key.fd) + self._writers.discard(key.fd) + return key + + def _select(self, r, w, timeout=None): + """ Wrapper for select.select because timeout is a positional arg """ + return select.select(r, w, [], timeout) + + def select(self, timeout=None): + # Selecting on empty lists on Windows errors out. + if not len(self._readers) and not len(self._writers): + return [] + + timeout = None if timeout is None else max(timeout, 0.0) + ready = [] + r, w, _ = _syscall_wrapper( + self._select, True, self._readers, self._writers, timeout + ) + r = set(r) + w = set(w) + for fd in r | w: + events = 0 + if fd in r: + events |= EVENT_READ + if fd in w: + events |= EVENT_WRITE + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + +if hasattr(select, "poll"): + + class PollSelector(BaseSelector): + """ Poll-based selector """ + + def __init__(self): + super(PollSelector, self).__init__() + self._poll = select.poll() + + def register(self, fileobj, events, data=None): + key = super(PollSelector, self).register(fileobj, events, data) + event_mask = 0 + if events & EVENT_READ: + event_mask |= select.POLLIN + if events & EVENT_WRITE: + event_mask |= select.POLLOUT + self._poll.register(key.fd, event_mask) + return key + + def unregister(self, fileobj): + key = super(PollSelector, self).unregister(fileobj) + self._poll.unregister(key.fd) + return key + + def _wrap_poll(self, timeout=None): + """ Wrapper function for select.poll.poll() so that + _syscall_wrapper can work with only seconds. """ + if timeout is not None: + if timeout <= 0: + timeout = 0 + else: + # select.poll.poll() has a resolution of 1 millisecond, + # round away from zero to wait *at least* timeout seconds. + timeout = math.ceil(timeout * 1e3) + result = self._poll.poll(timeout) + return result + + def select(self, timeout=None): + ready = [] + fd_events = _syscall_wrapper( + self._wrap_poll, True, timeout=timeout + ) + for fd, event_mask in fd_events: + events = 0 + if event_mask & ~select.POLLIN: + events |= EVENT_WRITE + if event_mask & ~select.POLLOUT: + events |= EVENT_READ + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + +if hasattr(select, "epoll"): + + class EpollSelector(BaseSelector): + """ Epoll-based selector """ + + def __init__(self): + super(EpollSelector, self).__init__() + self._epoll = select.epoll() + + def fileno(self): + return self._epoll.fileno() + + def register(self, fileobj, events, data=None): + key = super(EpollSelector, self).register(fileobj, events, data) + events_mask = 0 + if events & EVENT_READ: + events_mask |= select.EPOLLIN + if events & EVENT_WRITE: + events_mask |= select.EPOLLOUT + _syscall_wrapper(self._epoll.register, False, key.fd, events_mask) + return key + + def unregister(self, fileobj): + key = super(EpollSelector, self).unregister(fileobj) + try: + _syscall_wrapper(self._epoll.unregister, False, key.fd) + except SelectorError: + # This can occur when the fd was closed since registry. + pass + return key + + def select(self, timeout=None): + if timeout is not None: + if timeout <= 0: + timeout = 0.0 + else: + # select.epoll.poll() has a resolution of 1 millisecond + # but luckily takes seconds so we don't need a wrapper + # like PollSelector. Just for better rounding. + timeout = math.ceil(timeout * 1e3) * 1e-3 + timeout = float(timeout) + else: + timeout = -1.0 # epoll.poll() must have a float. + # We always want at least 1 to ensure that select can be called + # with no file descriptors registered. Otherwise will fail. + max_events = max(len(self._fd_to_key), 1) + ready = [] + fd_events = _syscall_wrapper( + self._epoll.poll, True, timeout=timeout, maxevents=max_events + ) + for fd, event_mask in fd_events: + events = 0 + if event_mask & ~select.EPOLLIN: + events |= EVENT_WRITE + if event_mask & ~select.EPOLLOUT: + events |= EVENT_READ + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + def close(self): + self._epoll.close() + super(EpollSelector, self).close() + + +if hasattr(select, "kqueue"): + + class KqueueSelector(BaseSelector): + """ Kqueue / Kevent-based selector """ + + def __init__(self): + super(KqueueSelector, self).__init__() + self._kqueue = select.kqueue() + + def fileno(self): + return self._kqueue.fileno() + + def register(self, fileobj, events, data=None): + key = super(KqueueSelector, self).register(fileobj, events, data) + if events & EVENT_READ: + kevent = select.kevent( + key.fd, select.KQ_FILTER_READ, select.KQ_EV_ADD + ) + _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) + if events & EVENT_WRITE: + kevent = select.kevent( + key.fd, select.KQ_FILTER_WRITE, select.KQ_EV_ADD + ) + _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) + return key + + def unregister(self, fileobj): + key = super(KqueueSelector, self).unregister(fileobj) + if key.events & EVENT_READ: + kevent = select.kevent( + key.fd, select.KQ_FILTER_READ, select.KQ_EV_DELETE + ) + try: + _syscall_wrapper( + self._kqueue.control, False, [kevent], 0, 0 + ) + except SelectorError: + pass + if key.events & EVENT_WRITE: + kevent = select.kevent( + key.fd, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE + ) + try: + _syscall_wrapper( + self._kqueue.control, False, [kevent], 0, 0 + ) + except SelectorError: + pass + return key + + def select(self, timeout=None): + if timeout is not None: + timeout = max(timeout, 0) + max_events = len(self._fd_to_key) * 2 + ready_fds = {} + kevent_list = _syscall_wrapper( + self._kqueue.control, True, None, max_events, timeout + ) + for kevent in kevent_list: + fd = kevent.ident + event_mask = kevent.filter + events = 0 + if event_mask == select.KQ_FILTER_READ: + events |= EVENT_READ + if event_mask == select.KQ_FILTER_WRITE: + events |= EVENT_WRITE + key = self._key_from_fd(fd) + if key: + if key.fd not in ready_fds: + ready_fds[key.fd] = (key, events & key.events) + else: + old_events = ready_fds[key.fd][1] + ready_fds[key.fd] = ( + key, (events | old_events) & key.events + ) + return list(ready_fds.values()) + + def close(self): + self._kqueue.close() + super(KqueueSelector, self).close() + + +if not hasattr(select, 'select'): # Platform-specific: AppEngine + HAS_SELECT = False + + +def _can_allocate(struct): + """ Checks that select structs can be allocated by the underlying + operating system, not just advertised by the select module. We don't + check select() because we'll be hopeful that most platforms that + don't have it available will not advertise it. (ie: GAE) """ + try: + # select.poll() objects won't fail until used. + if struct == 'poll': + p = select.poll() + p.poll(0) + # All others will fail on allocation. + else: + getattr(select, struct)().close() + return True + + except (OSError, AttributeError) as e: + return False + + + + +# Choose the best implementation, roughly: +# kqueue == epoll > poll > select. Devpoll not supported. (See above) +# select() also can't accept a FD > FD_SETSIZE (usually around 1024) +def DefaultSelector(): + """ This function serves as a first call for DefaultSelector to + detect if the select module is being monkey-patched incorrectly + by eventlet, greenlet, and preserve proper behavior. """ + global _DEFAULT_SELECTOR + if _DEFAULT_SELECTOR is None: + if _can_allocate('kqueue'): + _DEFAULT_SELECTOR = KqueueSelector + elif _can_allocate('epoll'): + _DEFAULT_SELECTOR = EpollSelector + elif _can_allocate('poll'): + _DEFAULT_SELECTOR = PollSelector + elif hasattr(select, 'select'): + _DEFAULT_SELECTOR = SelectSelector + else: # Platform-specific: AppEngine + raise ValueError('Platform does not have a selector') + + return _DEFAULT_SELECTOR() diff --git a/requests3/core/_http/util/ssl_.py b/requests3/core/_http/util/ssl_.py new file mode 100644 index 00000000..73369f80 --- /dev/null +++ b/requests3/core/_http/util/ssl_.py @@ -0,0 +1,389 @@ +from __future__ import absolute_import +import errno +import logging +import warnings +import hmac + +from binascii import hexlify, unhexlify +from hashlib import md5, sha1, sha256 + +from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning +from ..packages.ssl_match_hostname import ( + match_hostname as _match_hostname, CertificateError +) + +SSLContext = None +HAS_SNI = False +IS_PYOPENSSL = False +IS_SECURETRANSPORT = False +# Maps the length of a digest to a possible hash function producing this digest +HASHFUNC_MAP = {32: md5, 40: sha1, 64: sha256} +log = logging.getLogger(__name__) + + +def _const_compare_digest_backport(a, b): + """ + Compare two digests of equal length in constant time. + + The digests must be of type str/bytes. + Returns True if the digests match, and False otherwise. + """ + result = abs(len(a) - len(b)) + for l, r in zip(bytearray(a), bytearray(b)): + result |= l ^ r + return result == 0 + + +_const_compare_digest = getattr( + hmac, 'compare_digest', _const_compare_digest_backport +) +try: # Test for SSL features + import ssl + from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23 + from ssl import HAS_SNI # Has SNI? + from ssl import SSLError as BaseSSLError +except ImportError: + + class BaseSSLError(Exception): + pass + + +try: + from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION +except ImportError: + OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000 + OP_NO_COMPRESSION = 0x20000 +# A secure default. +# Sources for more information on TLS ciphers: +# +# - https://wiki.mozilla.org/Security/Server_Side_TLS +# - https://www.ssllabs.com/projects/best-practices/index.html +# - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ +# +# The general intent is: +# - Prefer TLS 1.3 cipher suites +# - prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE), +# - prefer ECDHE over DHE for better performance, +# - prefer any AES-GCM and ChaCha20 over any AES-CBC for better performance and +# security, +# - prefer AES-GCM over ChaCha20 because hardware-accelerated AES is common, +# - disable NULL authentication, MD5 MACs and DSS for security reasons. +DEFAULT_CIPHERS = ':'.join( + [ + 'TLS13-AES-256-GCM-SHA384', + 'TLS13-CHACHA20-POLY1305-SHA256', + 'TLS13-AES-128-GCM-SHA256', + 'ECDH+AESGCM', + 'ECDH+CHACHA20', + 'DH+AESGCM', + 'DH+CHACHA20', + 'ECDH+AES256', + 'DH+AES256', + 'ECDH+AES128', + 'DH+AES', + 'RSA+AESGCM', + 'RSA+AES', + '!aNULL', + '!eNULL', + '!MD5', + ] +) +try: + from ssl import SSLContext # Modern SSL? +except ImportError: + + # TODO: Can we remove this by choosing to support only platforms with + # actual SSLContext objects? + class SSLContext(object): # Platform-specific: Python 2 & 3.1 + + def __init__(self, protocol_version): + self.protocol = protocol_version + # Use default values from a real SSLContext + self.check_hostname = False + self.verify_mode = ssl.CERT_NONE + self.ca_certs = None + self.options = 0 + self.certfile = None + self.keyfile = None + self.ciphers = None + + def load_cert_chain(self, certfile, keyfile): + self.certfile = certfile + self.keyfile = keyfile + + def load_verify_locations(self, cafile=None, capath=None): + self.ca_certs = cafile + if capath is not None: + raise SSLError("CA directories not supported in older Pythons") + + def set_ciphers(self, cipher_suite): + self.ciphers = cipher_suite + + def wrap_socket(self, socket, server_hostname=None, server_side=False): + warnings.warn( + 'A true SSLContext object is not available. This prevents ' + 'urllib3 from configuring SSL appropriately and may cause ' + 'certain SSL connections to fail. You can upgrade to a newer ' + 'version of Python to solve this. For more information, see ' + 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' + '#ssl-warnings', + InsecurePlatformWarning, + ) + kwargs = { + 'keyfile': self.keyfile, + 'certfile': self.certfile, + 'ca_certs': self.ca_certs, + 'cert_reqs': self.verify_mode, + 'ssl_version': self.protocol, + 'server_side': server_side, + } + return wrap_socket(socket, ciphers=self.ciphers, **kwargs) + + +def assert_fingerprint(cert, fingerprint): + """ + Checks if given fingerprint matches the supplied certificate. + + :param cert: + Certificate as bytes object. + :param fingerprint: + Fingerprint as string of hexdigits, can be interspersed by colons. + """ + fingerprint = fingerprint.replace(':', '').lower() + digest_length = len(fingerprint) + hashfunc = HASHFUNC_MAP.get(digest_length) + if not hashfunc: + raise SSLError( + 'Fingerprint of invalid length: {0}'.format(fingerprint) + ) + + # We need encode() here for py32; works on py2 and p33. + fingerprint_bytes = unhexlify(fingerprint.encode()) + cert_digest = hashfunc(cert).digest() + if not _const_compare_digest(cert_digest, fingerprint_bytes): + raise SSLError( + 'Fingerprints did not match. Expected "{0}", got "{1}".'.format( + fingerprint, hexlify(cert_digest) + ) + ) + + +def resolve_cert_reqs(candidate): + """ + Resolves the argument to a numeric constant, which can be passed to + the wrap_socket function/method from the ssl module. + Defaults to :data:`ssl.CERT_NONE`. + If given a string it is assumed to be the name of the constant in the + :mod:`ssl` module or its abbrevation. + (So you can specify `REQUIRED` instead of `CERT_REQUIRED`. + If it's neither `None` nor a string we assume it is already the numeric + constant which can directly be passed to wrap_socket. + """ + if candidate is None: + return CERT_NONE + + if isinstance(candidate, str): + res = getattr(ssl, candidate, None) + if res is None: + res = getattr(ssl, 'CERT_' + candidate) + return res + + return candidate + + +def resolve_ssl_version(candidate): + """ + like resolve_cert_reqs + """ + if candidate is None: + return PROTOCOL_SSLv23 + + if isinstance(candidate, str): + res = getattr(ssl, candidate, None) + if res is None: + res = getattr(ssl, 'PROTOCOL_' + candidate) + return res + + return candidate + + +def create_urllib3_context( + ssl_version=None, cert_reqs=None, options=None, ciphers=None +): + """All arguments have the same meaning as ``ssl_wrap_socket``. + + By default, this function does a lot of the same work that + ``ssl.create_default_context`` does on Python 3.4+. It: + + - Disables SSLv2, SSLv3, and compression + - Sets a restricted set of server ciphers + + If you wish to enable SSLv3, you can do:: + + from urllib3.util import ssl_ + context = ssl_.create_urllib3_context() + context.options &= ~ssl_.OP_NO_SSLv3 + + You can do the same to enable compression (substituting ``COMPRESSION`` + for ``SSLv3`` in the last line above). + + :param ssl_version: + The desired protocol version to use. This will default to + PROTOCOL_SSLv23 which will negotiate the highest protocol that both + the server and your installation of OpenSSL support. + :param cert_reqs: + Whether to require the certificate verification. This defaults to + ``ssl.CERT_REQUIRED``. + :param options: + Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``, + ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``. + :param ciphers: + Which cipher suites to allow the server to select. + :returns: + Constructed SSLContext object with specified options + :rtype: SSLContext + """ + context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23) + # Setting the default here, as we may have no ssl module on import + cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs + if options is None: + options = 0 + # SSLv2 is easily broken and is considered harmful and dangerous + options |= OP_NO_SSLv2 + # SSLv3 has several problems and is now dangerous + options |= OP_NO_SSLv3 + # Disable compression to prevent CRIME attacks for OpenSSL 1.0+ + # (issue #309) + options |= OP_NO_COMPRESSION + context.options |= options + context.set_ciphers(ciphers or DEFAULT_CIPHERS) + context.verify_mode = cert_reqs + if getattr( + context, 'check_hostname', None + ) is not None: # Platform-specific: Python 3.2 + # We do our own verification, including fingerprints and alternative + # hostnames. So disable it here + context.check_hostname = False + return context + + +def merge_context_settings( + context, + keyfile=None, + certfile=None, + cert_reqs=None, + ca_certs=None, + ca_cert_dir=None, +): + """ + Merges provided settings into an SSL Context. + """ + if cert_reqs is not None: + context.verify_mode = resolve_cert_reqs(cert_reqs) + if ca_certs or ca_cert_dir: + try: + context.load_verify_locations(ca_certs, ca_cert_dir) + except IOError as e: # Platform-specific: Python 2.6, 2.7, 3.2 + raise SSLError(e) + + # Py33 raises FileNotFoundError which subclasses OSError + # These are not equivalent unless we check the errno attribute + except OSError as e: # Platform-specific: Python 3.3 and beyond + if e.errno == errno.ENOENT: + raise SSLError(e) + + raise + + elif getattr(context, 'load_default_certs', None) is not None: + # try to load OS default certs; works well on Windows (require Python3.4+) + context.load_default_certs() + if certfile: + context.load_cert_chain(certfile, keyfile) + return context + + +def ssl_wrap_socket( + sock, + keyfile=None, + certfile=None, + cert_reqs=None, + ca_certs=None, + server_hostname=None, + ssl_version=None, + ciphers=None, + ssl_context=None, + ca_cert_dir=None, +): + """ + All arguments except for server_hostname, ssl_context, and ca_cert_dir have + the same meaning as they do when using :func:`ssl.wrap_socket`. + + :param server_hostname: + When SNI is supported, the expected hostname of the certificate + :param ssl_context: + A pre-made :class:`SSLContext` object. If none is provided, one will + be created using :func:`create_urllib3_context`. + :param ciphers: + A string of ciphers we wish the client to support. + :param ca_cert_dir: + A directory containing CA certificates in multiple separate files, as + supported by OpenSSL's -CApath flag or the capath argument to + SSLContext.load_verify_locations(). + """ + context = ssl_context + if context is None: + # Note: This branch of code and all the variables in it are no longer + # used by urllib3 itself. We should consider deprecating and removing + # this code. + context = create_urllib3_context( + ssl_version, cert_reqs, ciphers=ciphers + ) + if ca_certs or ca_cert_dir: + try: + context.load_verify_locations(ca_certs, ca_cert_dir) + except IOError as e: # Platform-specific: Python 2.6, 2.7, 3.2 + raise SSLError(e) + + # Py33 raises FileNotFoundError which subclasses OSError + # These are not equivalent unless we check the errno attribute + except OSError as e: # Platform-specific: Python 3.3 and beyond + if e.errno == errno.ENOENT: + raise SSLError(e) + + raise + + elif getattr(context, 'load_default_certs', None) is not None: + # try to load OS default certs; works well on Windows (require Python3.4+) + context.load_default_certs() + if certfile: + context.load_cert_chain(certfile, keyfile) + if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI + return context.wrap_socket(sock, server_hostname=server_hostname) + + warnings.warn( + 'An HTTPS request has been made, but the SNI (Server Name ' + 'Indication) extension to TLS is not available on this platform. ' + 'This may cause the server to present an incorrect TLS ' + 'certificate, which can cause validation failures. You can upgrade to ' + 'a newer version of Python to solve this. For more information, see ' + 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' + '#ssl-warnings', + SNIMissingWarning, + ) + return context.wrap_socket(sock) + + +def match_hostname(cert, asserted_hostname): + try: + _match_hostname(cert, asserted_hostname) + except CertificateError as e: + log.error( + 'Certificate did not match expected hostname: %s. ' + 'Certificate: %s', + asserted_hostname, + cert, + ) + # Add cert to exception and reraise so client code can inspect + # the cert when catching the exception, if they want to + e._peer_cert = cert + raise diff --git a/requests3/core/_http/util/timeout.py b/requests3/core/_http/util/timeout.py new file mode 100644 index 00000000..35d49520 --- /dev/null +++ b/requests3/core/_http/util/timeout.py @@ -0,0 +1,261 @@ +from __future__ import absolute_import + +# The default socket timeout, used by httplib to indicate that no timeout was +# specified by the user +from socket import _GLOBAL_DEFAULT_TIMEOUT +import time + +from ..exceptions import TimeoutStateError + +# A sentinel value to indicate that no timeout was specified by the user in +# urllib3 +_Default = object() +# Use time.monotonic if available. +current_time = getattr(time, "monotonic", time.time) + + +class Timeout(object): + """ Timeout configuration. + + Timeouts can be defined as a default for a pool:: + + timeout = Timeout(connect=2.0, read=7.0) + http = PoolManager(timeout=timeout) + response = http.request('GET', 'http://example.com/') + + Or per-request (which overrides the default for the pool):: + + response = http.request('GET', 'http://example.com/', timeout=Timeout(10)) + + Timeouts can be disabled by setting all the parameters to ``None``:: + + no_timeout = Timeout(connect=None, read=None) + response = http.request('GET', 'http://example.com/, timeout=no_timeout) + + + :param total: + This combines the connect and read timeouts into one; the read timeout + will be set to the time leftover from the connect attempt. In the + event that both a connect timeout and a total are specified, or a read + timeout and a total are specified, the shorter timeout will be applied. + + Defaults to None. + + :type total: integer, float, or None + + :param connect: + The maximum amount of time to wait for a connection attempt to a server + to succeed. Omitting the parameter will default the connect timeout to + the system default, probably `the global default timeout in socket.py + `_. + None will set an infinite timeout for connection attempts. + + :type connect: integer, float, or None + + :param read: + The maximum amount of time to wait between consecutive + read operations for a response from the server. Omitting + the parameter will default the read timeout to the system + default, probably `the global default timeout in socket.py + `_. + None will set an infinite timeout. + + :type read: integer, float, or None + + .. note:: + + Many factors can affect the total amount of time for urllib3 to return + an HTTP response. + + For example, Python's DNS resolver does not obey the timeout specified + on the socket. Other factors that can affect total request time include + high CPU load, high swap, the program running at a low priority level, + or other behaviors. + + In addition, the read and total timeouts only measure the time between + read operations on the socket connecting the client and the server, + not the total amount of time for the request to return a complete + response. For most requests, the timeout is raised because the server + has not sent the first byte in the specified time. This is not always + the case; if a server streams one byte every fifteen seconds, a timeout + of 20 seconds will not trigger, even though the request will take + several minutes to complete. + + If your goal is to cut off any request after a set amount of wall clock + time, consider having a second "watcher" thread to cut off a slow + request. + """ + # : A sentinel object representing the default timeout value + DEFAULT_TIMEOUT = _GLOBAL_DEFAULT_TIMEOUT + + def __init__(self, total=None, connect=_Default, read=_Default): + self._connect = self._validate_timeout(connect, 'connect') + self._read = self._validate_timeout(read, 'read') + self.total = self._validate_timeout(total, 'total') + self._start_connect = None + + def __str__(self): + return '%s(connect=%r, read=%r, total=%r)' % ( + type(self).__name__, self._connect, self._read, self.total + ) + + @classmethod + def _validate_timeout(cls, value, name): + """ Check that a timeout attribute is valid. + + :param value: The timeout value to validate + :param name: The name of the timeout attribute to validate. This is + used to specify in error messages. + :return: The validated and casted version of the given value. + :raises ValueError: If it is a numeric value less than or equal to + zero, or the type is not an integer, float, or None. + """ + if value is _Default: + return cls.DEFAULT_TIMEOUT + + if value is None or value is cls.DEFAULT_TIMEOUT: + return value + + if isinstance(value, bool): + raise ValueError( + "Timeout cannot be a boolean value. It must " + "be an int, float or None." + ) + + try: + float(value) + except (TypeError, ValueError): + raise ValueError( + "Timeout value %s was %s, but it must be an " + "int, float or None." % (name, value) + ) + + try: + if value <= 0: + raise ValueError( + "Attempted to set %s timeout to %s, but the " + "timeout cannot be set to a value less " + "than or equal to 0." % (name, value) + ) + + except TypeError: # Python 3 + raise ValueError( + "Timeout value %s was %s, but it must be an " + "int, float or None." % (name, value) + ) + + return value + + @classmethod + def from_float(cls, timeout): + """ Create a new Timeout from a legacy timeout value. + + The timeout value used by httplib.py sets the same timeout on the + connect(), and recv() socket requests. This creates a :class:`Timeout` + object that sets the individual timeouts to the ``timeout`` value + passed to this function. + + :param timeout: The legacy timeout value. + :type timeout: integer, float, sentinel default object, or None + :return: Timeout object + :rtype: :class:`Timeout` + """ + return Timeout(read=timeout, connect=timeout) + + def clone(self): + """ Create a copy of the timeout object + + Timeout properties are stored per-pool but each request needs a fresh + Timeout object to ensure each one has its own start/stop configured. + + :return: a copy of the timeout object + :rtype: :class:`Timeout` + """ + # We can't use copy.deepcopy because that will also create a new object + # for _GLOBAL_DEFAULT_TIMEOUT, which socket.py uses as a sentinel to + # detect the user default. + return Timeout( + connect=self._connect, read=self._read, total=self.total + ) + + def start_connect(self): + """ Start the timeout clock, used during a connect() attempt + + :raises urllib3.exceptions.TimeoutStateError: if you attempt + to start a timer that has been started already. + """ + if self._start_connect is not None: + raise TimeoutStateError("Timeout timer has already been started.") + + self._start_connect = current_time() + return self._start_connect + + def get_connect_duration(self): + """ Gets the time elapsed since the call to :meth:`start_connect`. + + :return: Elapsed time. + :rtype: float + :raises urllib3.exceptions.TimeoutStateError: if you attempt + to get duration for a timer that hasn't been started. + """ + if self._start_connect is None: + raise TimeoutStateError( + "Can't get connect duration for timer " "that has not started." + ) + + return current_time() - self._start_connect + + @property + def connect_timeout(self): + """ Get the value to use when setting a connection timeout. + + This will be a positive float or integer, the value None + (never timeout), or the default system timeout. + + :return: Connect timeout. + :rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None + """ + if self.total is None: + return self._connect + + if self._connect is None or self._connect is self.DEFAULT_TIMEOUT: + return self.total + + return min(self._connect, self.total) + + @property + def read_timeout(self): + """ Get the value for the read timeout. + + This assumes some time has elapsed in the connection timeout and + computes the read timeout appropriately. + + If self.total is set, the read timeout is dependent on the amount of + time taken by the connect timeout. If the connection time has not been + established, a :exc:`~urllib3.exceptions.TimeoutStateError` will be + raised. + + :return: Value to use for the read timeout. + :rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None + :raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect` + has not yet been called on this object. + """ + if ( + self.total is not None and + self.total is not self.DEFAULT_TIMEOUT and + self._read is not None and + self._read is not self.DEFAULT_TIMEOUT + ): + # In case the connect timeout has not yet been established. + if self._start_connect is None: + return self._read + + return max( + 0, min(self.total - self.get_connect_duration(), self._read) + ) + + elif self.total is not None and self.total is not self.DEFAULT_TIMEOUT: + return max(0, self.total - self.get_connect_duration()) + + else: + return self._read diff --git a/requests3/core/_http/util/url.py b/requests3/core/_http/util/url.py new file mode 100644 index 00000000..f4c6a745 --- /dev/null +++ b/requests3/core/_http/util/url.py @@ -0,0 +1,221 @@ +from __future__ import absolute_import +from collections import namedtuple + +from ..exceptions import LocationParseError + +url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'] +# We only want to normalize urls with an HTTP(S) scheme. +# urllib3 infers URLs without a scheme (None) to be http. +NORMALIZABLE_SCHEMES = ('http', 'https', None) + + +class Url(namedtuple('Url', url_attrs)): + """ + Datastructure for representing an HTTP URL. Used as a return value for + :func:`parse_url`. Both the scheme and host are normalized as they are + both case-insensitive according to RFC 3986. + """ + __slots__ = () + + def __new__( + cls, + scheme=None, + auth=None, + host=None, + port=None, + path=None, + query=None, + fragment=None, + ): + if path and not path.startswith('/'): + path = '/' + path + if scheme: + scheme = scheme.lower() + if host and scheme in NORMALIZABLE_SCHEMES: + host = host.lower() + return super(Url, cls).__new__( + cls, scheme, auth, host, port, path, query, fragment + ) + + @property + def hostname(self): + """For backwards-compatibility with urlparse. We're nice like that.""" + return self.host + + @property + def request_uri(self): + """Absolute path including the query string.""" + uri = self.path or '/' + if self.query is not None: + uri += '?' + self.query + return uri + + @property + def netloc(self): + """Network location including host and port""" + if self.port: + return '%s:%d' % (self.host, self.port) + + return self.host + + @property + def url(self): + """ + Convert self into a url + + This function should more or less round-trip with :func:`.parse_url`. The + returned url may not be exactly the same as the url inputted to + :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls + with a blank port will have : removed). + + Example: :: + + >>> U = parse_url('http://google.com/mail/') + >>> U.url + 'http://google.com/mail/' + >>> Url('http', 'username:password', 'host.com', 80, + ... '/path', 'query', 'fragment').url + 'http://username:password@host.com:80/path?query#fragment' + """ + scheme, auth, host, port, path, query, fragment = self + url = '' + # We use "is not None" we want things to happen with empty strings (or 0 port) + if scheme is not None: + url += scheme + '://' + if auth is not None: + url += auth + '@' + if host is not None: + url += host + if port is not None: + url += ':' + str(port) + if path is not None: + url += path + if query is not None: + url += '?' + query + if fragment is not None: + url += '#' + fragment + return url + + def __str__(self): + return self.url + + +def split_first(s, delims): + """ + Given a string and an iterable of delimiters, split on the first found + delimiter. Return two split parts and the matched delimiter. + + If not found, then the first part is the full input string. + + Example:: + + >>> split_first('foo/bar?baz', '?/=') + ('foo', 'bar?baz', '/') + >>> split_first('foo/bar?baz', '123') + ('foo/bar?baz', '', None) + + Scales linearly with number of delims. Not ideal for large number of delims. + """ + min_idx = None + min_delim = None + for d in delims: + idx = s.find(d) + if idx < 0: + continue + + if min_idx is None or idx < min_idx: + min_idx = idx + min_delim = d + if min_idx is None or min_idx < 0: + return s, '', None + + return s[:min_idx], s[min_idx + 1:], min_delim + + +def parse_url(url): + """ + Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is + performed to parse incomplete urls. Fields not provided will be None. + + Partly backwards-compatible with :mod:`urlparse`. + + Example:: + + >>> parse_url('http://google.com/mail/') + Url(scheme='http', host='google.com', port=None, path='/mail/', ...) + >>> parse_url('google.com:80') + Url(scheme=None, host='google.com', port=80, path=None, ...) + >>> parse_url('/foo?bar') + Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...) + """ + # While this code has overlap with stdlib's urlparse, it is much + # simplified for our needs and less annoying. + # Additionally, this implementations does silly things to be optimal + # on CPython. + if not url: + # Empty + return Url() + + scheme = None + auth = None + host = None + port = None + path = None + fragment = None + query = None + # Scheme + if '://' in url: + scheme, url = url.split('://', 1) + # Find the earliest Authority Terminator + # (http://tools.ietf.org/html/rfc3986#section-3.2) + url, path_, delim = split_first(url, ['/', '?', '#']) + if delim: + # Reassemble the path + path = delim + path_ + # Auth + if '@' in url: + # Last '@' denotes end of auth part + auth, url = url.rsplit('@', 1) + # IPv6 + if url and url[0] == '[': + host, url = url.split(']', 1) + host += ']' + # Port + if ':' in url: + _host, port = url.split(':', 1) + if not host: + host = _host + if port: + # If given, ports must be integers. No whitespace, no plus or + # minus prefixes, no non-integer digits such as ^2 (superscript). + if not port.isdigit(): + raise LocationParseError(url) + + try: + port = int(port) + except ValueError: + raise LocationParseError(url) + + else: + # Blank ports are cool, too. (rfc3986#section-3.2.3) + port = None + elif not host and url: + host = url + if not path: + return Url(scheme, auth, host, port, path, query, fragment) + + # Fragment + if '#' in path: + path, fragment = path.split('#', 1) + # Query + if '?' in path: + path, query = path.split('?', 1) + return Url(scheme, auth, host, port, path, query, fragment) + + +def get_host(url): + """ + Deprecated. Use :func:`parse_url` instead. + """ + p = parse_url(url) + return p.scheme or 'http', p.hostname, p.port diff --git a/requests3/core/_http/util/wait.py b/requests3/core/_http/util/wait.py new file mode 100644 index 00000000..155bba0e --- /dev/null +++ b/requests3/core/_http/util/wait.py @@ -0,0 +1,39 @@ +from .selectors import (HAS_SELECT, DefaultSelector, EVENT_READ, EVENT_WRITE) + + +def _wait_for_io_events(socks, events, timeout=None): + """ Waits for IO events to be available from a list of sockets + or optionally a single socket if passed in. Returns a list of + sockets that can be interacted with immediately. """ + if not HAS_SELECT: + raise ValueError('Platform does not have a selector') + + if not isinstance(socks, list): + # Probably just a single socket. + if hasattr(socks, "fileno"): + socks = [socks] + # Otherwise it might be a non-list iterable. + else: + socks = list(socks) + with DefaultSelector() as selector: + for sock in socks: + selector.register(sock, events) + return [ + key[0].fileobj + for key in selector.select(timeout) + if key[1] & events + ] + + +def wait_for_read(socks, timeout=None): + """ Waits for reading to be available from a list of sockets + or optionally a single socket if passed in. Returns a list of + sockets that can be read from immediately. """ + return _wait_for_io_events(socks, EVENT_READ, timeout) + + +def wait_for_write(socks, timeout=None): + """ Waits for writing to be available from a list of sockets + or optionally a single socket if passed in. Returns a list of + sockets that can be written to immediately. """ + return _wait_for_io_events(socks, EVENT_WRITE, timeout) diff --git a/requests3/core/api.py b/requests3/core/api.py new file mode 100644 index 00000000..1b692636 --- /dev/null +++ b/requests3/core/api.py @@ -0,0 +1,53 @@ +import trio + +from ._http import AsyncPoolManager, PoolManager +from ._http._backends import TrioBackend + +async def request( + method, + url, + timeout, + *, + body=None, + headers=None, + preload_content=False, + pool=None, + **kwargs +): + """Returns a Response object, to be awaited.""" + if not pool: + pool = AsyncPoolManager(backend=TrioBackend()) + return await pool.urlopen( + method=method, + url=url, + headers=headers, + preload_content=preload_content, + body=body, + **kwargs + ) + + +def blocking_request( + method, + url, + timeout, + *, + body=None, + headers=None, + preload_content=False, + pool=None, + **kwargs +): + """Returns a Response object.""" + if not pool: + pool = PoolManager() + with pool as http: + r = http.urlopen( + method=method, + url=url, + headers=headers, + preload_content=preload_content, + body=body, + **kwargs + ) + return r diff --git a/requests/exceptions.py b/requests3/exceptions.py similarity index 86% rename from requests/exceptions.py rename to requests3/exceptions.py index a80cad80..af4b9ba3 100644 --- a/requests/exceptions.py +++ b/requests3/exceptions.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.exceptions ~~~~~~~~~~~~~~~~~~~ @@ -19,8 +18,11 @@ class RequestException(IOError): response = kwargs.pop('response', None) self.response = response self.request = kwargs.pop('request', None) - if (response is not None and not self.request and - hasattr(response, 'request')): + if ( + response is not None and + not self.request and + hasattr(response, 'request') + ): self.request = self.response.request super(RequestException, self).__init__(*args, **kwargs) @@ -69,12 +71,12 @@ class TooManyRedirects(RequestException): """Too many redirects.""" -class MissingSchema(RequestException, ValueError): - """The URL schema (e.g. http or https) is missing.""" +class MissingScheme(RequestException, ValueError): + """The URL scheme (e.g. http or https) is missing.""" -class InvalidSchema(RequestException, ValueError): - """See defaults.py for valid schemas.""" +class InvalidScheme(RequestException, ValueError): + """See defaults.py for valid schemes.""" class InvalidURL(RequestException, ValueError): @@ -108,9 +110,14 @@ class RetryError(RequestException): class UnrewindableBodyError(RequestException): """Requests encountered an error when trying to rewind a body""" + +class InvalidBodyError(RequestException, ValueError): + """An invalid request body was specified""" + + + + # Warnings - - class RequestsWarning(Warning): """Base warning for Requests.""" pass diff --git a/requests3/help.py b/requests3/help.py new file mode 100644 index 00000000..ebab0f4d --- /dev/null +++ b/requests3/help.py @@ -0,0 +1,101 @@ +"""Module containing bug report helper(s).""" +from __future__ import print_function + +import json +import platform +import sys +import ssl + +import idna +import urllib3 +import chardet + +from . import types + +from . import __version__ as requests_version + +try: + from urllib3.contrib import pyopenssl +except ImportError: + pyopenssl = None + OpenSSL = None + cryptography = None +else: + import OpenSSL + import cryptography + + +def _implementation() -> types.Help: + """Return a dict with the Python implementation and version. + + Provide both the name and the version of the Python implementation + currently running. For example, on CPython 2.7.5 it will return + {'name': 'CPython', 'version': '2.7.5'}. + + This function works best on CPython and PyPy: in particular, it probably + doesn't work for Jython or IronPython. Future investigation should be done + to work out the correct shape of the code for those platforms. + """ + implementation = platform.python_implementation() + if implementation == "CPython": + implementation_version = platform.python_version() + elif implementation == "PyPy": + implementation_version = "%s.%s.%s" % ( + sys.pypy_version_info.major, + sys.pypy_version_info.minor, + sys.pypy_version_info.micro, + ) + if sys.pypy_version_info.releaselevel != "final": + implementation_version = "".join( + [implementation_version, sys.pypy_version_info.releaselevel] + ) + elif implementation == "Jython": + implementation_version = platform.python_version() # Complete Guess + elif implementation == "IronPython": + implementation_version = platform.python_version() # Complete Guess + else: + implementation_version = "Unknown" + return {"name": implementation, "version": implementation_version} + + +def info() -> types.Help: + """Generate information for a bug report.""" + try: + platform_info = {"system": platform.system(), "release": platform.release()} + except IOError: + platform_info = {"system": "Unknown", "release": "Unknown"} + implementation_info = _implementation() + urllib3_info = {"version": urllib3.__version__} + chardet_info = {"version": chardet.__version__} + pyopenssl_info = {"version": None, "openssl_version": ""} + if OpenSSL: + pyopenssl_info = { + "version": OpenSSL.__version__, + "openssl_version": "%x" % OpenSSL.SSL.OPENSSL_VERSION_NUMBER, + } + cryptography_info = {"version": getattr(cryptography, "__version__", "")} + idna_info = {"version": getattr(idna, "__version__", "")} + + system_ssl = ssl.OPENSSL_VERSION_NUMBER + system_ssl_info = {"version": "%x" % system_ssl if system_ssl is not None else ""} + return { + "platform": platform_info, + "implementation": implementation_info, + "system_ssl": system_ssl_info, + "using_pyopenssl": pyopenssl is not None, + "pyOpenSSL": pyopenssl_info, + "urllib3": urllib3_info, + "chardet": chardet_info, + "cryptography": cryptography_info, + "idna": idna_info, + "requests": {"version": requests_version}, + } + + +def main(): + """Pretty-print the bug information as JSON.""" + print(json.dumps(info(), sort_keys=True, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/requests/hooks.py b/requests3/hooks.py similarity index 100% rename from requests/hooks.py rename to requests3/hooks.py index 7a51f212..0a2a4dc7 100644 --- a/requests/hooks.py +++ b/requests3/hooks.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.hooks ~~~~~~~~~~~~~~ @@ -17,9 +16,10 @@ HOOKS = ['response'] def default_hooks(): return {event: [] for event in HOOKS} + + + # TODO: response is the only one - - def dispatch_hook(key, hooks, hook_data, **kwargs): """Dispatches a hook dictionary on a given piece of data.""" hooks = hooks or {} diff --git a/requests/models.py b/requests3/models.py similarity index 59% rename from requests/models.py rename to requests3/models.py index 62dcd0b7..35c114ac 100644 --- a/requests/models.py +++ b/requests3/models.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.models ~~~~~~~~~~~~~~~ @@ -8,75 +7,97 @@ This module contains the primary objects that power Requests. """ import datetime +import codecs import sys # Import encoding now, to avoid implicit import later. # Implicit import within threads may cause LookupError when standard library is in a ZIP, # such as in Embedded Python. See https://github.com/requests/requests/issues/3578. +import rfc3986 import encodings.idna -from urllib3.fields import RequestField -from urllib3.filepost import encode_multipart_formdata -from urllib3.util import parse_url -from urllib3.exceptions import ( - DecodeError, ReadTimeoutError, ProtocolError, LocationParseError) +from .core._http.fields import RequestField +from .core._http.filepost import encode_multipart_formdata +from .core._http.exceptions import ( + DecodeError, ReadTimeoutError, ProtocolError, LocationParseError +) from io import UnsupportedOperation from .hooks import default_hooks from .structures import CaseInsensitiveDict +import requests3 as requests from .auth import HTTPBasicAuth from .cookies import cookiejar_from_dict, get_cookie_header, _copy_cookie_jar from .exceptions import ( - HTTPError, MissingSchema, InvalidURL, ChunkedEncodingError, - ContentDecodingError, ConnectionError, StreamConsumedError) + HTTPError, + MissingScheme, + InvalidURL, + ChunkedEncodingError, + ContentDecodingError, + ConnectionError, + StreamConsumedError, + InvalidHeader, + InvalidBodyError, + ReadTimeout, +) from ._internal_utils import to_native_string, unicode_is_ascii from .utils import ( - guess_filename, get_auth_from_url, requote_uri, - stream_decode_response_unicode, to_key_val_list, parse_header_links, - iter_slices, guess_json_utf, super_len, check_header_validity) -from .compat import ( - Callable, Mapping, - cookielib, urlunparse, urlsplit, urlencode, str, bytes, - is_py2, chardet, builtin_str, basestring) -from .compat import json as complexjson + guess_filename, + get_auth_from_url, + requote_uri, + stream_decode_response_unicode, + to_key_val_list, + parse_header_links, + iter_slices, + guess_json_utf, + super_len, + check_header_validity, + is_stream, +) +from .basics import ( + cookielib, + urlunparse, + urlsplit, + urlencode, + str, + bytes, + chardet, + builtin_str, + basestring, +) +import json as complexjson from .status_codes import codes -#: The set of HTTP status codes that indicate an automatically +# : The set of HTTP status codes that indicate an automatically #: processable redirect. REDIRECT_STATI = ( - codes.moved, # 301 - codes.found, # 302 - codes.other, # 303 - codes.temporary_redirect, # 307 - codes.permanent_redirect, # 308 + codes['moved'], # 301 + codes['found'], # 302 + codes['other'], # 303 + codes['temporary_redirect'], # 307 + codes['permanent_redirect'], # 308 ) - DEFAULT_REDIRECT_LIMIT = 30 CONTENT_CHUNK_SIZE = 10 * 1024 ITER_CHUNK_SIZE = 512 class RequestEncodingMixin(object): + @property def path_url(self): """Build the path URL to use.""" - url = [] - p = urlsplit(self.url) - path = p.path if not path: path = '/' - url.append(path) - query = p.query if query: url.append('?') url.append(query) - return ''.join(url) @staticmethod @@ -87,11 +108,12 @@ class RequestEncodingMixin(object): 2-tuples. Order is retained if data is a list of 2-tuples but arbitrary if parameters are supplied as a dict. """ - if isinstance(data, (str, bytes)): return data + elif hasattr(data, 'read'): return data + elif hasattr(data, '__iter__'): result = [] for k, vs in to_key_val_list(data): @@ -100,9 +122,13 @@ class RequestEncodingMixin(object): for v in vs: if v is not None: result.append( - (k.encode('utf-8') if isinstance(k, str) else k, - v.encode('utf-8') if isinstance(v, str) else v)) + ( + k.encode('utf-8') if isinstance(k, str) else k, + v.encode('utf-8') if isinstance(v, str) else v, + ) + ) return urlencode(result, doseq=True) + else: return data @@ -118,13 +144,13 @@ class RequestEncodingMixin(object): """ if (not files): raise ValueError("Files must be provided.") + elif isinstance(data, basestring): raise ValueError("Data must not be a string.") new_fields = [] fields = to_key_val_list(data or {}) files = to_key_val_list(files or {}) - for field, val in fields: if isinstance(val, basestring) or not hasattr(val, '__iter__'): val = [val] @@ -133,11 +159,14 @@ class RequestEncodingMixin(object): # Don't call str() on bytestrings: in Py3 it all goes wrong. if not isinstance(v, bytes): v = str(v) - new_fields.append( - (field.decode('utf-8') if isinstance(field, bytes) else field, - v.encode('utf-8') if isinstance(v, str) else v)) - + ( + field.decode('utf-8') if isinstance( + field, bytes + ) else field, + v.encode('utf-8') if isinstance(v, str) else v, + ) + ) for (k, v) in files: # support for explicit filename ft = None @@ -152,7 +181,6 @@ class RequestEncodingMixin(object): else: fn = guess_filename(v) or k fp = v - if isinstance(fp, (str, bytes, bytearray)): fdata = fp elif hasattr(fp, 'read'): @@ -165,18 +193,18 @@ class RequestEncodingMixin(object): rf = RequestField(name=k, data=fdata, filename=fn, headers=fh) rf.make_multipart(content_type=ft) new_fields.append(rf) - body, content_type = encode_multipart_formdata(new_fields) - return body, content_type class RequestHooksMixin(object): + def register_hook(self, event, hook): """Properly register a hook.""" - if event not in self.hooks: - raise ValueError('Unsupported event specified, with event name "%s"' % (event)) + raise ValueError( + 'Unsupported event specified, with event name "%s"' % (event) + ) if isinstance(hook, Callable): self.hooks[event].append(hook) @@ -187,10 +215,10 @@ class RequestHooksMixin(object): """Deregister a previously registered hook. Returns True if the hook existed, False if not. """ - try: self.hooks[event].remove(hook) return True + except ValueError: return False @@ -222,22 +250,41 @@ class Request(RequestHooksMixin): >>> req.prepare() """ + __slots__ = ( + 'method', + 'url', + 'headers', + 'files', + 'data', + 'params', + 'auth', + 'cookies', + 'hooks', + 'json', + ) - def __init__(self, - method=None, url=None, headers=None, files=None, data=None, - params=None, auth=None, cookies=None, hooks=None, json=None): - + def __init__( + self, + method=None, + url=None, + headers=None, + files=None, + data=None, + params=None, + auth=None, + cookies=None, + hooks=None, + json=None, + ): # Default empty dicts for dict params. data = [] if data is None else data files = [] if files is None else files headers = {} if headers is None else headers params = {} if params is None else params hooks = {} if hooks is None else hooks - self.hooks = default_hooks() for (k, v) in list(hooks.items()): self.register_hook(event=k, hook=v) - self.method = method self.url = url self.headers = headers @@ -286,44 +333,60 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): >>> s.send(r) """ + __slots__ = ( + 'method', + 'url', + 'headers', + '_cookies', + 'body', + 'hooks', + '_body_position', + ) def __init__(self): - #: HTTP verb to send to the server. + # : HTTP verb to send to the server. self.method = None - #: HTTP URL to send the request to. + # : HTTP URL to send the request to. self.url = None - #: dictionary of HTTP headers. + # : dictionary of HTTP headers. self.headers = None # The `CookieJar` used to create the Cookie header will be stored here # after prepare_cookies is called self._cookies = None - #: request body to send to the server. + # : request body to send to the server. self.body = None - #: dictionary of callback hooks, for internal usage. + # : dictionary of callback hooks, for internal usage. self.hooks = default_hooks() - #: integer denoting starting position of a readable file-like body. + # : integer denoting starting position of a readable file-like body. self._body_position = None - def prepare(self, - method=None, url=None, headers=None, files=None, data=None, - params=None, auth=None, cookies=None, hooks=None, json=None): + def prepare( + self, + method=None, + url=None, + headers=None, + files=None, + data=None, + params=None, + auth=None, + cookies=None, + hooks=None, + json=None, + ): """Prepares the entire request with the given parameters.""" - self.prepare_method(method) self.prepare_url(url, params) self.prepare_headers(headers) self.prepare_cookies(cookies) self.prepare_body(data, files, json) self.prepare_auth(auth, url) - # Note that prepare_auth must be last to enable authentication schemes # such as OAuth to work on a fully prepared request. - # This MUST go after prepare_auth. Authenticators could add a hook self.prepare_hooks(hooks) def __repr__(self): - return '' % (self.method) + return f'' def copy(self): p = PreparedRequest() @@ -339,8 +402,10 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): def prepare_method(self, method): """Prepares the given HTTP method.""" self.method = method - if self.method is not None: - self.method = to_native_string(self.method.upper()) + if self.method is None: + raise ValueError('Request method cannot be "None"') + + self.method = to_native_string(self.method.upper()) @staticmethod def _get_idna_encoded_host(host): @@ -350,11 +415,12 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): host = idna.encode(host, uts46=True).decode('utf-8') except idna.IDNAError: raise UnicodeError + return host - def prepare_url(self, url, params): + def prepare_url(self, url, params, validate=False): """Prepares the given HTTP URL.""" - #: Accept objects that have string representations. + # : Accept objects that have string representations. #: We're unable to blindly call unicode/str functions #: as this will include the bytestring indicator (b'') #: on python 3.x. @@ -362,11 +428,9 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): if isinstance(url, bytes): url = url.decode('utf8') else: - url = unicode(url) if is_py2 else str(url) - - # Remove leading whitespaces from url - url = url.lstrip() - + url = str(url) + # Ignore any leading and trailing whitespace characters. + url = url.strip() # Don't do any URL preparation for non-HTTP schemes like `mailto`, # `data` etc to work around exceptions from `url_parse`, which # handles RFC 3986 only. @@ -376,71 +440,54 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): # Support for unicode domain names and paths. try: - scheme, auth, host, port, path, query, fragment = parse_url(url) - except LocationParseError as e: - raise InvalidURL(*e.args) + uri = rfc3986.urlparse(url) + if validate: + rfc3986.normalize_uri(url) + except rfc3986.exceptions.RFC3986Exception: + raise InvalidURL(f"Invalid URL {url!r}: URL is imporoper.") - if not scheme: - error = ("Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?") + if not uri.scheme: + error = ( + "Invalid URL {0!r}: No scheme supplied. Perhaps you meant http://{0}?" + ) error = error.format(to_native_string(url, 'utf8')) + raise MissingScheme(error) - raise MissingSchema(error) - - if not host: - raise InvalidURL("Invalid URL %r: No host supplied" % url) + if not uri.host: + raise InvalidURL(f"Invalid URL {url!r}: No host supplied") # In general, we want to try IDNA encoding the hostname if the string contains # non-ASCII characters. This allows users to automatically get the correct IDNA # behaviour. For strings containing only ASCII characters, we need to also verify # it doesn't start with a wildcard (*), before allowing the unencoded hostname. - if not unicode_is_ascii(host): + if not unicode_is_ascii(uri.host): try: - host = self._get_idna_encoded_host(host) + uri = uri.copy_with(host=self._get_idna_encoded_host(uri.host)) except UnicodeError: raise InvalidURL('URL has an invalid label.') - elif host.startswith(u'*'): + + elif uri.host.startswith(u'*'): raise InvalidURL('URL has an invalid label.') - # Carefully reconstruct the network location - netloc = auth or '' - if netloc: - netloc += '@' - netloc += host - if port: - netloc += ':' + str(port) - # Bare domains aren't valid URLs. - if not path: - path = '/' - - if is_py2: - if isinstance(scheme, str): - scheme = scheme.encode('utf-8') - if isinstance(netloc, str): - netloc = netloc.encode('utf-8') - if isinstance(path, str): - path = path.encode('utf-8') - if isinstance(query, str): - query = query.encode('utf-8') - if isinstance(fragment, str): - fragment = fragment.encode('utf-8') - + if not uri.path: + uri = uri.copy_with(path='/') if isinstance(params, (str, bytes)): params = to_native_string(params) - enc_params = self._encode_params(params) if enc_params: - if query: - query = '%s&%s' % (query, enc_params) + if uri.query: + uri = uri.copy_with(query=f'{uri.query}&{enc_params}') else: - query = enc_params - - url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment])) - self.url = url + uri = uri.copy_with(query=enc_params) + # url = requote_uri( + # urlunparse([uri.scheme, uri.authority, uri.path, None, uri.query, uri.fragment]) + # ) + # Normalize the URI. + self.url = rfc3986.normalize_uri(uri.unsplit()) def prepare_headers(self, headers): """Prepares the given HTTP headers.""" - self.headers = CaseInsensitiveDict() if headers: for header in headers.items(): @@ -451,14 +498,11 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): def prepare_body(self, data, files, json=None): """Prepares the given HTTP body data.""" - # Check if file, fo, generator, iterator. # If not, run through normal process. - # Nottin' on you. body = None content_type = None - if not data and json is not None: # urllib3 requires a bytes-like body. Python 2's json.dumps # provides this natively, but Python 3 gives a Unicode string. @@ -479,7 +523,6 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): if is_stream: body = data - if getattr(body, 'tell', None) is not None: # Record the current file position before reading. # This will allow us to rewind a file in the event @@ -490,14 +533,11 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): # This differentiates from None, allowing us to catch # a failed `tell()` later when trying to rewind the body self._body_position = object() - if files: - raise NotImplementedError('Streamed bodies and files are mutually exclusive.') + raise NotImplementedError( + 'Streamed bodies and files are mutually exclusive.' + ) - if length: - self.headers['Content-Length'] = builtin_str(length) - else: - self.headers['Transfer-Encoding'] = 'chunked' else: # Multi-part file uploads. if files: @@ -509,47 +549,58 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): content_type = None else: content_type = 'application/x-www-form-urlencoded' - - self.prepare_content_length(body) - # Add content-type if it wasn't explicitly provided. if content_type and ('content-type' not in self.headers): self.headers['Content-Type'] = content_type - + self.prepare_content_length(body) self.body = body def prepare_content_length(self, body): - """Prepare Content-Length header based on request method and body""" + """Prepares Content-Length header. + + If the length of the body of the request can be computed, Content-Length + is set using ``super_len``. If user has manually set either a + Transfer-Encoding or Content-Length header when it should not be set + (they should be mutually exclusive) an InvalidHeader + error will be raised. + """ if body is not None: length = super_len(body) if length: - # If length exists, set it. Otherwise, we fallback - # to Transfer-Encoding: chunked. self.headers['Content-Length'] = builtin_str(length) - elif self.method not in ('GET', 'HEAD') and self.headers.get('Content-Length') is None: + elif is_stream(body): + self.headers['Transfer-Encoding'] = 'chunked' + else: + raise InvalidBodyError( + 'Non-null body must have length or be streamable.' + ) + + elif self.method not in ('GET', 'HEAD') and self.headers.get( + 'Content-Length' + ) is None: # Set Content-Length to 0 for methods that can have a body # but don't provide one. (i.e. not GET or HEAD) self.headers['Content-Length'] = '0' + if 'Transfer-Encoding' in self.headers and 'Content-Length' in self.headers: + raise InvalidHeader( + 'Conflicting Headers: Both Transfer-Encoding and ' + 'Content-Length are set.' + ) def prepare_auth(self, auth, url=''): """Prepares the given HTTP auth data.""" - # If no Auth is explicitly provided, extract it from the URL first. if auth is None: url_auth = get_auth_from_url(self.url) auth = url_auth if any(url_auth) else None - if auth: if isinstance(auth, tuple) and len(auth) == 2: # special-case basic HTTP auth auth = HTTPBasicAuth(*auth) - # Allow auth to make its changes. r = auth(self) - # Update self to reflect the auth changes. self.__dict__.update(r.__dict__) - # Recompute Content-Length self.prepare_content_length(self.body) @@ -568,7 +619,6 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): self._cookies = cookies else: self._cookies = cookiejar_from_dict(cookies) - cookie_header = get_cookie_header(self._cookies, self) if cookie_header is not None: self.headers['Cookie'] = cookie_header @@ -582,61 +632,67 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): for event in hooks: self.register_hook(event, hooks[event]) + def send(self, session=None, **send_kwargs): + """Sends the PreparedRequest to the given Session. + If none is provided, one is created for you.""" + session = requests.Session() if session is None else session + with session: + return session.send(self, **send_kwargs) + class Response(object): """The :class:`Response ` object, which contains a server's response to an HTTP request. """ - __attrs__ = [ - '_content', 'status_code', 'headers', 'url', 'history', - 'encoding', 'reason', 'cookies', 'elapsed', 'request' + '_content', + 'status_code', + 'headers', + 'url', + 'history', + 'encoding', + 'reason', + 'cookies', + 'elapsed', + 'request', ] + __slots__ = __attrs__ + ['_content_consumed', 'raw', '_next', 'connection'] def __init__(self): self._content = False self._content_consumed = False self._next = None - - #: Integer Code of responded HTTP Status, e.g. 404 or 200. + # : Integer Code of responded HTTP Status, e.g. 404 or 200. self.status_code = None - - #: Case-insensitive Dictionary of Response Headers. + # : Case-insensitive Dictionary of Response Headers. #: For example, ``headers['content-encoding']`` will return the #: value of a ``'Content-Encoding'`` response header. self.headers = CaseInsensitiveDict() - - #: File-like object representation of response (for advanced usage). + # : File-like object representation of response (for advanced usage). #: Use of ``raw`` requires that ``stream=True`` be set on the request. # This requirement does not apply for use internally to Requests. self.raw = None - - #: Final URL location of Response. + # : Final URL location of Response. self.url = None - - #: Encoding to decode with when accessing r.text. + # : Encoding to decode with when accessing r.text or + #: r.iter_content(decode_unicode=True) self.encoding = None - - #: A list of :class:`Response ` objects from + # : A list of :class:`Response ` objects from #: the history of the Request. Any redirect responses will end #: up here. The list is sorted from the oldest to the most recent request. self.history = [] - - #: Textual reason of responded HTTP Status, e.g. "Not Found" or "OK". + # : Textual reason of responded HTTP Status, e.g. "Not Found" or "OK". self.reason = None - - #: A CookieJar of Cookies the server sent back. + # : A CookieJar of Cookies the server sent back. self.cookies = cookiejar_from_dict({}) - - #: The amount of time elapsed between sending the request + # : The amount of time elapsed between sending the request #: and the arrival of the response (as a timedelta). #: This property specifically measures the time taken between sending #: the first byte of the request and finishing parsing the headers. It #: is therefore unaffected by consuming the response content or the #: value of the ``stream`` keyword argument. self.elapsed = datetime.timedelta(0) - - #: The :class:`PreparedRequest ` object to which this + # : The :class:`PreparedRequest ` object to which this #: is a response. self.request = None @@ -651,13 +707,11 @@ class Response(object): # sure the content has been fully read. if not self._content_consumed: self.content - return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): for name, value in state.items(): setattr(self, name, value) - # pickled objects do not have .raw setattr(self, '_content_consumed', True) setattr(self, 'raw', None) @@ -665,26 +719,6 @@ class Response(object): def __repr__(self): return '' % (self.status_code) - def __bool__(self): - """Returns True if :attr:`status_code` is less than 400. - - This attribute checks if the status code of the response is between - 400 and 600 to see if there was a client error or a server error. If - the status code, is between 200 and 400, this will return True. This - is **not** a check to see if the response code is ``200 OK``. - """ - return self.ok - - def __nonzero__(self): - """Returns True if :attr:`status_code` is less than 400. - - This attribute checks if the status code of the response is between - 400 and 600 to see if there was a client error or a server error. If - the status code, is between 200 and 400, this will return True. This - is **not** a check to see if the response code is ``200 OK``. - """ - return self.ok - def __iter__(self): """Allows you to use a response as an iterator.""" return self.iter_content(128) @@ -702,6 +736,7 @@ class Response(object): self.raise_for_status() except HTTPError: return False + return True @property @@ -709,12 +744,19 @@ class Response(object): """True if this Response is a well-formed HTTP redirect that could have been processed automatically (by :meth:`Session.resolve_redirects`). """ - return ('location' in self.headers and self.status_code in REDIRECT_STATI) + return ( + 'location' in self.headers and self.status_code in REDIRECT_STATI + ) @property def is_permanent_redirect(self): """True if this Response one of the permanent versions of redirect.""" - return ('location' in self.headers and self.status_code in (codes.moved_permanently, codes.permanent_redirect)) + return ( + 'location' in self.headers and + self.status_code in ( + codes.moved_permanently, codes.permanent_redirect + ) + ) @property def next(self): @@ -726,7 +768,7 @@ class Response(object): """The apparent encoding, provided by the chardet library.""" return chardet.detect(self.content)['encoding'] - def iter_content(self, chunk_size=1, decode_unicode=False): + def iter_content(self, decode_unicode=False): """Iterates over the response data. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. The chunk size is the number of bytes it should @@ -739,46 +781,70 @@ class Response(object): chunks are received. If stream=False, data is returned as a single chunk. - If decode_unicode is True, content will be decoded using the best - available encoding based on the response. + If using decode_unicode, the encoding must be set to a valid encoding + enumeration before invoking iter_content. """ + DEFAULT_CHUNK_SIZE = 1 + def generate(): # Special case for urllib3. if hasattr(self.raw, 'stream'): try: - for chunk in self.raw.stream(chunk_size, decode_content=True): + for chunk in self.raw.stream( + # chunk_size, decode_content=True + decode_content=True + ): yield chunk + except ProtocolError as e: - raise ChunkedEncodingError(e) + if self.headers.get('Transfer-Encoding') == 'chunked': + raise ChunkedEncodingError(e) + + else: + raise ConnectionError(e) + except DecodeError as e: raise ContentDecodingError(e) + except ReadTimeoutError as e: - raise ConnectionError(e) + raise ReadTimeout(e) + else: # Standard file-like object. while True: chunk = self.raw.read(chunk_size) if not chunk: break + yield chunk self._content_consumed = True if self._content_consumed and isinstance(self._content, bool): raise StreamConsumedError() - elif chunk_size is not None and not isinstance(chunk_size, int): - raise TypeError("chunk_size must be an int, it is instead a %s." % type(chunk_size)) - # simulate reading small chunks of the content - reused_chunks = iter_slices(self._content, chunk_size) + # elif chunk_size is not None and not isinstance(chunk_size, int): + # raise TypeError( + # f"chunk_size must be an int, it is instead a {type(chunk_size)}." + # ) + + # simulate reading small chunks of the content + reused_chunks = iter_slices(self._content, DEFAULT_CHUNK_SIZE) stream_chunks = generate() chunks = reused_chunks if self._content_consumed else stream_chunks - if decode_unicode: - chunks = stream_decode_response_unicode(chunks, self) + if self.encoding is None: + raise TypeError( + 'encoding must be set before consuming streaming ' + 'responses' + ) + # check encoding value here, don't wait for the generator to be + # consumed before raising an exception + codecs.lookup(self.encoding) + chunks = stream_decode_response_unicode(chunks, self) return chunks def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=False, delimiter=None): @@ -788,24 +854,66 @@ class Response(object): .. note:: This method is not reentrant safe. """ - + carriage_return = u'\r' if decode_unicode else b'\r' + line_feed = u'\n' if decode_unicode else b'\n' pending = None + last_chunk_ends_with_cr = False + for chunk in self.iter_content( + chunk_size=chunk_size, decode_unicode=decode_unicode + ): + # Skip any null responses: if there is pending data it is necessarily an + # incomplete chunk, so if we don't have more data we don't want to bother + # trying to get it. Unconsumed pending data will be yielded anyway in the + # end of the loop if the stream ends. + if not chunk: + continue - for chunk in self.iter_content(chunk_size=chunk_size, decode_unicode=decode_unicode): - + # Consume any pending data if pending is not None: chunk = pending + chunk - + pending = None + # Either split on a line, or split on a specified delimiter if delimiter: lines = chunk.split(delimiter) else: + # Python splitlines() supports the universal newline (PEP 278). + # That means, '\r', '\n', and '\r\n' are all treated as end of + # line. If the last chunk ends with '\r', and the current chunk + # starts with '\n', they should be merged and treated as only + # *one* new line separator '\r\n' by splitlines(). + # This rule only applies when splitlines() is used. + # The last chunk ends with '\r', so the '\n' at chunk[0] + # is just the second half of a '\r\n' pair rather than a + # new line break. Just skip it. + skip_first_char = last_chunk_ends_with_cr and chunk.startswith( + line_feed + ) + last_chunk_ends_with_cr = chunk.endswith(carriage_return) + if skip_first_char: + chunk = chunk[1:] + # it's possible that after stripping the '\n' then chunk becomes empty + if not chunk: + continue + lines = chunk.splitlines() - - if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]: + # Calling `.split(delimiter)` will always end with whatever text + # remains beyond the delimiter, or '' if the delimiter is the end + # of the text. On the other hand, `.splitlines()` doesn't include + # a '' if the text ends in a line delimiter. + # + # For example: + # + # 'abc\ndef\n'.split('\n') ~> ['abc', 'def', ''] + # 'abc\ndef\n'.splitlines() ~> ['abc', 'def'] + # + # So if we have a specified delimiter, we always pop the final + # item and prepend it to the next chunk. + # + # If we're using `splitlines()`, we only do this if the chunk + # ended midway through a line. + incomplete_line = lines[-1] and lines[-1][-1] == chunk[-1] + if delimiter or incomplete_line: pending = lines.pop() - else: - pending = None - for line in lines: yield line @@ -815,18 +923,23 @@ class Response(object): @property def content(self): """Content of the response, in bytes.""" - if self._content is False: # Read the contents. if self._content_consumed: raise RuntimeError( - 'The content for this response was already consumed') + 'The content for this response was already consumed' + ) if self.status_code == 0 or self.raw is None: self._content = None else: - self._content = b''.join(self.iter_content(CONTENT_CHUNK_SIZE)) or b'' - + # self._content = await self.iter_content(CONTENT_CHUNK_SIZE) + # print(bytes().join( + # [await self.iter_content(CONTENT_CHUNK_SIZE)] + # )) + self._content = bytes().join( + self.iter_content() + ) or bytes() self._content_consumed = True # don't need to release the connection; that's been handled by urllib3 # since we exhausted the data. @@ -844,18 +957,15 @@ class Response(object): non-HTTP knowledge to make a better guess at the encoding, you should set ``r.encoding`` appropriately before accessing this property. """ - # Try charset from content-type content = None encoding = self.encoding - if not self.content: return str('') # Fallback to auto-detected encoding. if self.encoding is None: encoding = self.apparent_encoding - # Decode unicode from given encoding. try: content = str(self.content, encoding, errors='replace') @@ -867,7 +977,6 @@ class Response(object): # # So we try blindly encoding. content = str(self.content, errors='replace') - return content def json(self, **kwargs): @@ -876,7 +985,6 @@ class Response(object): :param \*\*kwargs: Optional arguments that ``json.loads`` takes. :raises ValueError: If the response body does not contain valid json. """ - if not self.encoding and self.content and len(self.content) > 3: # No encoding set. JSON RFC 4627 section 3 states we should expect # UTF-8, -16 or -32. Detect which one to use; If the detection or @@ -885,9 +993,11 @@ class Response(object): encoding = guess_json_utf(self.content) if encoding is not None: try: + content = self.content return complexjson.loads( - self.content.decode(encoding), **kwargs + content.decode(encoding), **kwargs ) + except UnicodeDecodeError: # Wrong UTF codec detected; usually because it's not UTF-8 # but some other 8-bit codec. This is an RFC violation, @@ -899,24 +1009,19 @@ class Response(object): @property def links(self): """Returns the parsed header links of the response, if any.""" - header = self.headers.get('link') - # l = MultiDict() l = {} - if header: links = parse_header_links(header) - for link in links: key = link.get('rel') or link.get('url') l[key] = link - return l def raise_for_status(self): - """Raises stored :class:`HTTPError`, if one occurred.""" - + """Raises stored :class:`HTTPError`, if one occurred. + Otherwise, returns the response object (self).""" http_error_msg = '' if isinstance(self.reason, bytes): # We attempt to decode utf-8 first because some servers @@ -929,16 +1034,19 @@ class Response(object): reason = self.reason.decode('iso-8859-1') else: reason = self.reason - if 400 <= self.status_code < 500: - http_error_msg = u'%s Client Error: %s for url: %s' % (self.status_code, reason, self.url) - + http_error_msg = u'%s Client Error: %s for url: %s' % ( + self.status_code, reason, self.url + ) elif 500 <= self.status_code < 600: - http_error_msg = u'%s Server Error: %s for url: %s' % (self.status_code, reason, self.url) - + http_error_msg = u'%s Server Error: %s for url: %s' % ( + self.status_code, reason, self.url + ) if http_error_msg: raise HTTPError(http_error_msg, response=self) + return self + def close(self): """Releases the connection back to the pool. Once this method has been called the underlying ``raw`` object must not be accessed again. @@ -947,7 +1055,178 @@ class Response(object): """ if not self._content_consumed: self.raw.close() - release_conn = getattr(self.raw, 'release_conn', None) if release_conn is not None: release_conn() + + +class AsyncResponse(Response): + def __init__(self, *args, **kwargs): + super(AsyncResponse, self).__init__(*args, **kwargs) + + async def json(self, **kwargs): + r"""Returns the json-encoded content of a response, if any. + + :param \*\*kwargs: Optional arguments that ``json.loads`` takes. + :raises ValueError: If the response body does not contain valid json. + """ + if not self.encoding and await self.content and len(await self.content) > 3: + # No encoding set. JSON RFC 4627 section 3 states we should expect + # UTF-8, -16 or -32. Detect which one to use; If the detection or + # decoding fails, fall back to `self.text` (using chardet to make + # a best guess). + encoding = guess_json_utf(await self.content) + if encoding is not None: + try: + content = await self.content + return complexjson.loads( + content.decode(encoding), **kwargs + ) + + except UnicodeDecodeError: + # Wrong UTF codec detected; usually because it's not UTF-8 + # but some other 8-bit codec. This is an RFC violation, + # and the server didn't bother to tell us what codec *was* + # used. + pass + return complexjson.loads(await self.text, **kwargs) + + @property + async def text(self): + """Content of the response, in unicode. + + If Response.encoding is None, encoding will be guessed using + ``chardet``. + + The encoding of the response content is determined based solely on HTTP + headers, following RFC 2616 to the letter. If you can take advantage of + non-HTTP knowledge to make a better guess at the encoding, you should + set ``r.encoding`` appropriately before accessing this property. + """ + # Try charset from content-type + content = None + encoding = self.encoding + if not await self.content: + return str('') + + # Fallback to auto-detected encoding. + if self.encoding is None: + encoding = self.apparent_encoding + # Decode unicode from given encoding. + try: + content = str(self.content, encoding, errors='replace') + except (LookupError, TypeError): + # A LookupError is raised if the encoding was not found which could + # indicate a misspelling or similar mistake. + # + # A TypeError can be raised if encoding is None + # + # So we try blindly encoding. + content = str(await self.content, errors='replace') + return content + + @property + async def content(self): + """Content of the response, in bytes.""" + if self._content is False: + # Read the contents. + if self._content_consumed: + raise RuntimeError( + 'The content for this response was already consumed' + ) + + if self.status_code == 0 or self.raw is None: + self._content = None + else: + # self._content = await self.iter_content(CONTENT_CHUNK_SIZE) + # print(bytes().join( + # [await self.iter_content(CONTENT_CHUNK_SIZE)] + # )) + self._content = bytes().join( + [await self.iter_content()] + ) or bytes() + self._content_consumed = True + # don't need to release the connection; that's been handled by urllib3 + # since we exhausted the data. + return self._content + + + @property + async def apparent_encoding(self): + """The apparent encoding, provided by the chardet library.""" + return chardet.detect(await self.content)['encoding'] + + async def iter_content(self, decode_unicode=False): + """Iterates over the response data. When stream=True is set on the + request, this avoids reading the content at once into memory for + large responses. The chunk size is the number of bytes it should + read into memory. This is not necessarily the length of each item + returned as decoding can take place. + + chunk_size must be of type int or None. A value of None will + function differently depending on the value of `stream`. + stream=True will read data as it arrives in whatever size the + chunks are received. If stream=False, data is returned as + a single chunk. + + If using decode_unicode, the encoding must be set to a valid encoding + enumeration before invoking iter_content. + """ + + DEFAULT_CHUNK_SIZE = 1 + + async def generate(): + # Special case for requests.core. + if hasattr(self.raw, 'stream'): + try: + async for chunk in self.raw.stream( + decode_content=True + ): + yield chunk + + except ProtocolError as e: + if self.headers.get('Transfer-Encoding') == 'chunked': + raise ChunkedEncodingError(e) + + else: + raise ConnectionError(e) + + except DecodeError as e: + raise ContentDecodingError(e) + + except ReadTimeoutError as e: + raise ReadTimeout(e) + + else: + # Standard file-like object. + while True: + chunk = await self.raw.read(chunk_size) + if not chunk: + break + + yield chunk + + self._content_consumed = True + + if self._content_consumed and isinstance(self._content, bool): + raise StreamConsumedError() + + reused_chunks = iter_slices(self._content, DEFAULT_CHUNK_SIZE) + try: + stream_chunks = await generate().__anext__() + except StopAsyncIteration: + stream_chunks = None + + chunks = reused_chunks if self._content_consumed else stream_chunks + if decode_unicode: + if self.encoding is None: + raise TypeError( + 'encoding must be set before consuming streaming ' + 'responses' + ) + + # check encoding value here, don't wait for the generator to be + # consumed before raising an exception + codecs.lookup(self.encoding) + chunks = stream_decode_response_unicode(chunks, self) + return chunks diff --git a/requests3/sessions.py b/requests3/sessions.py new file mode 100644 index 00000000..3c6168b1 --- /dev/null +++ b/requests3/sessions.py @@ -0,0 +1,947 @@ +# -*- coding: utf-8 -*- +""" +requests.session +~~~~~~~~~~~~~~~~ + +This module provides a Session object to manage and persist settings across +requests (cookies, auth, proxies). +""" +import os +import sys +import time +from collections import Mapping, OrderedDict +from datetime import timedelta + +from .core._http._backends.trio_backend import TrioBackend + +from .auth import _basic_auth_str +from .basics import cookielib, urljoin, urlparse, str +from .cookies import ( + cookiejar_from_dict, + extract_cookies_to_jar, + RequestsCookieJar, + merge_cookies, + _copy_cookie_jar, +) +from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT +from .hooks import default_hooks, dispatch_hook +from ._internal_utils import to_native_string +from .utils import to_key_val_list, default_headers, DEFAULT_PORTS +from .exceptions import ( + TooManyRedirects, + InvalidScheme, + ChunkedEncodingError, + ConnectionError, + ContentDecodingError, + InvalidHeader, +) + +from .structures import CaseInsensitiveDict +from .adapters import HTTPAdapter, AsyncHTTPAdapter + +from .utils import ( + requote_uri, + get_environ_proxies, + get_netrc_auth, + should_bypass_proxies, + get_auth_from_url, + is_valid_location, + rewind_body, +) + +from .status_codes import codes + +# formerly defined here, reexposed here for backward compatibility +from .models import REDIRECT_STATI + +# Preferred clock, based on which one is more accurate on a given system. +if sys.platform == "win32": + try: # Python 3.4+ + preferred_clock = time.perf_counter + except AttributeError: # Earlier than Python 3. + preferred_clock = time.clock +else: + preferred_clock = time.time + + +def merge_setting(request_setting, session_setting, dict_class=OrderedDict): + """Determines appropriate setting for a given request, taking into account + the explicit setting on that request, and the setting in the session. If a + setting is a dictionary, they will be merged together using `dict_class`. + """ + if session_setting is None: + return request_setting + + if request_setting is None: + return session_setting + + # Bypass if not a dictionary (e.g. verify) + if not ( + isinstance(session_setting, Mapping) and isinstance(request_setting, Mapping) + ): + return request_setting + + merged_setting = dict_class(to_key_val_list(session_setting)) + merged_setting.update(to_key_val_list(request_setting)) + # Remove keys that are set to None. Extract keys first to avoid altering + # the dictionary during iteration. + none_keys = [k for (k, v) in merged_setting.items() if v is None] + for key in none_keys: + del merged_setting[key] + return merged_setting + + +def merge_hooks(request_hooks, session_hooks, dict_class=OrderedDict): + """Properly merges both requests and session hooks. + + This is necessary because when request_hooks == {'response': []}, the + merge breaks Session hooks entirely. + """ + if session_hooks is None or session_hooks.get("response") == []: + return request_hooks + + if request_hooks is None or request_hooks.get("response") == []: + return session_hooks + + return merge_setting(request_hooks, session_hooks, dict_class) + + +class SessionRedirectMixin(object): + def get_redirect_target(self, response): + """Receives a Response. Returns a redirect URI or ``None``""" + # Due to the nature of how requests processes redirects this method will + # be called at least once upon the original response and at least twice + # on each subsequent redirect response (if any). + # If a custom mixin is used to handle this logic, it may be advantageous + # to cache the redirect location onto the response object as a private + # attribute. + if response.is_redirect: + if not is_valid_location(response): + raise InvalidHeader( + "Response contains multiple Location headers. " + "Unable to perform redirect." + ) + + location = response.headers["location"] + # Currently the underlying http module on py3 decode headers + # in latin1, but empirical evidence suggests that latin1 is very + # rarely used with non-ASCII characters in HTTP headers. + # It is more likely to get UTF8 header rather than latin1. + # This causes incorrect handling of UTF8 encoded location headers. + # To solve this, we re-encode the location in latin1. + location = location.encode("latin1") + return to_native_string(location, "utf8") + + return None + + def resolve_redirects( + self, + response, + request, + stream=False, + timeout=None, + verify=True, + cert=None, + proxies=None, + yield_requests=False, + **adapter_kwargs, + ): + """Given a Response, yields Responses until 'Location' header-based + redirection ceases, or the Session.max_redirects limit has been + reached. + """ + history = [ + response + ] # keep track of history; seed it with the original response + location_url = self.get_redirect_target(response) + while location_url: + prepared_request = request.copy() + try: + response.content # Consume socket so it can be released + except ( + ChunkedEncodingError, + ConnectionError, + ContentDecodingError, + RuntimeError, + ): + response.raw.read(decode_content=False) + if len(response.history) >= self.max_redirects: + raise TooManyRedirects( + "Exceeded %s redirects." % self.max_redirects, response=response + ) + + # Release the connection back into the pool. + response.close() + # Handle redirection without scheme (see: RFC 1808 Section 4) + if location_url.startswith("//"): + parsed_rurl = urlparse(response.url) + location_url = "%s:%s" % ( + to_native_string(parsed_rurl.scheme), + location_url, + ) + # The scheme should be lower case... + parsed = urlparse(location_url) + location_url = parsed.geturl() + # Facilitate relative 'location' headers, as allowed by RFC 7231. + # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') + # Compliant with RFC3986, we percent encode the url. + if not parsed.netloc: + location_url = urljoin(response.url, requote_uri(location_url)) + else: + location_url = requote_uri(location_url) + prepared_request.url = to_native_string(location_url) + method_changed = self.rebuild_method(prepared_request, response) + # https://github.com/kennethreitz/requests/issues/2590 + # If method is changed to GET we need to remove body and associated headers. + if method_changed and prepared_request.method == "GET": + # https://github.com/requests/requests/issues/3490 + purged_headers = ("Content-Length", "Content-Type", "Transfer-Encoding") + for header in purged_headers: + prepared_request.headers.pop(header, None) + prepared_request.body = None + headers = prepared_request.headers + try: + del headers["Cookie"] + except KeyError: + pass + # Extract any cookies sent on the response to the cookiejar + # in the new request. Because we've mutated our copied prepared + # request, use the old one that we haven't yet touched. + extract_cookies_to_jar(prepared_request._cookies, request, response.raw) + merge_cookies(prepared_request._cookies, self.cookies) + prepared_request.prepare_cookies(prepared_request._cookies) + # Rebuild auth and proxy information. + proxies = self.rebuild_proxies(prepared_request, proxies) + self.rebuild_auth(prepared_request, response) + # A failed tell() sets `_body_position` to `object()`. This non-None + # value ensures `rewindable` will be True, allowing us to raise an + # UnrewindableBodyError, instead of hanging the connection. + rewindable = prepared_request._body_position is not None and ( + "Content-Length" in headers or "Transfer-Encoding" in headers + ) + # Attempt to rewind consumed file-like object. + if rewindable: + rewind_body(prepared_request) + # Override the original request. + request = prepared_request + if yield_requests: + yield request + + else: + response = self.send( + request, + stream=stream, + timeout=timeout, + verify=verify, + cert=cert, + proxies=proxies, + allow_redirects=False, + **adapter_kwargs, + ) + # copy our history tracker into the response + response.history = history[:] + # append the new response to the history tracker for the next iteration + history.append(response) + extract_cookies_to_jar(self.cookies, prepared_request, response.raw) + # extract redirect url, if any, for the next loop + location_url = self.get_redirect_target(response) + yield response + + def rebuild_auth(self, prepared_request, response): + """When being redirected we may want to strip authentication from the + request to avoid leaking credentials. This method intelligently + removes + and reapplies authentication where possible to avoid credential loss. + """ + headers = prepared_request.headers + url = prepared_request.url + if "Authorization" in headers: + # If we get redirected to a new host, we should strip out any + # authentication headers. + original_parsed = urlparse(response.request.url) + redirect_parsed = urlparse(url) + if original_parsed.hostname != redirect_parsed.hostname: + del headers["Authorization"] + # .netrc might have more auth for us on our new host. + new_auth = get_netrc_auth(url) if self.trust_env else None + if new_auth is not None: + prepared_request.prepare_auth(new_auth) + return + + def rebuild_proxies(self, prepared_request, proxies): + """This method re-evaluates the proxy configuration by + considering the environment variables. If we are redirected to a + URL covered by NO_PROXY, we strip the proxy configuration. + Otherwise, we set missing proxy keys for this URL (in case they + were stripped by a previous redirect). + + This method also replaces the Proxy-Authorization header where + necessary. + + :rtype: dict + """ + proxies = proxies if proxies is not None else {} + headers = prepared_request.headers + url = prepared_request.url + scheme = urlparse(url).scheme + new_proxies = proxies.copy() + no_proxy = proxies.get("no_proxy") + bypass_proxy = should_bypass_proxies(url, no_proxy=no_proxy) + if self.trust_env and not bypass_proxy: + environ_proxies = get_environ_proxies(url, no_proxy=no_proxy) + proxy = environ_proxies.get(scheme, environ_proxies.get("all")) + if proxy: + new_proxies.setdefault(scheme, proxy) + if "Proxy-Authorization" in headers: + del headers["Proxy-Authorization"] + try: + username, password = get_auth_from_url(new_proxies[scheme]) + except KeyError: + username, password = None, None + if username and password: + headers["Proxy-Authorization"] = _basic_auth_str(username, password) + return new_proxies + + def rebuild_method(self, prepared_request, response): + """When being redirected we may want to change the method of the request + based on certain specs or browser behavior. + + :rtype bool: + :return: boolean expressing if the method changed during rebuild. + """ + method = original_method = prepared_request.method + # http://tools.ietf.org/html/rfc7231#section-6.4.4 + if response.status_code == codes.see_other and method != "HEAD": + method = "GET" + # If a POST is responded to with a 301 or 302, turn it into a GET. This has + # become a common pattern in browsers and was introduced into later versions + # of HTTP RFCs. While some browsers transform other methods to GET, little of + # that has been standardized. For that reason, we're using curl as a model + # which only supports POST->GET. + if response.status_code in (codes.found, codes.moved) and method == "POST": + method = "GET" + prepared_request.method = method + return method != original_method + + +class Session(SessionRedirectMixin): + """A Requests session. + + Provides cookie persistence, connection-pooling, and configuration. + + Basic Usage:: + + >>> import requests + >>> s = requests.Session() + >>> s.get('https://httpbin.org/get') + + + Or as a context manager:: + + >>> with requests.Session() as s: + >>> s.get('https://httpbin.org/get') + + """ + + __slots__ = [ + "headers", + "cookies", + "auth", + "proxies", + "hooks", + "params", + "verify", + "cert", + "prefetch", + "adapters", + "stream", + "trust_env", + "max_redirects", + ] + + __slots__ + + def __init__(self): + # : A case-insensitive dictionary of headers to be sent on each + #: :class:`Request ` sent from this + #: :class:`Session `. + self.headers = default_headers() + # : Default Authentication tuple or object to attach to + #: :class:`Request `. + self.auth = None + # : Dictionary mapping protocol or protocol and host to the URL of the proxy + #: (e.g. {'http': 'foo.bar:3128', 'http://host.name': 'foo.bar:4012'}) to + #: be used on each :class:`Request `. + self.proxies = {} + # : Event-handling hooks. + self.hooks = default_hooks() + # : Dictionary of querystring data to attach to each + #: :class:`Request `. The dictionary values may be lists for + #: representing multivalued query parameters. + self.params = {} + # : Stream response content default. + self.stream = False + # : SSL Verification default. + self.verify = True + # : SSL client certificate default, if String, path to ssl client + #: cert file (.pem). If Tuple, ('cert', 'key') pair. + self.cert = None + # : Maximum number of redirects allowed. If the request exceeds this + #: limit, a :class:`TooManyRedirects` exception is raised. + #: This defaults to requests.models.DEFAULT_REDIRECT_LIMIT, which is + #: 30. + self.max_redirects = DEFAULT_REDIRECT_LIMIT + # : Trust environment settings for proxy configuration, default + #: authentication and similar. + self.trust_env = True + # : A CookieJar containing all currently outstanding cookies set on this + #: session. By default it is a + #: :class:`RequestsCookieJar `, but + #: may be any other ``cookielib.CookieJar`` compatible object. + self.cookies = cookiejar_from_dict({}) + # Default connection adapters. + self.adapters = OrderedDict() + self.mount("https://", HTTPAdapter()) + self.mount("http://", HTTPAdapter()) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def prepare_request(self, request): + """Constructs a :class:`PreparedRequest ` for + transmission and returns it. The :class:`PreparedRequest` has settings + merged from the :class:`Request ` instance and those of the + :class:`Session`. + + :param request: :class:`Request` instance to prepare with this + Session's settings. + :rtype: requests.PreparedRequest + """ + cookies = request.cookies or {} + # Bootstrap CookieJar. + if not isinstance(cookies, cookielib.CookieJar): + cookies = cookiejar_from_dict(cookies) + # Merge with session cookies + session_cookies = _copy_cookie_jar(self.cookies) + merged_cookies = merge_cookies(session_cookies, cookies) + # Set environment's basic authentication if not explicitly set. + auth = request.auth + if self.trust_env and not auth and not self.auth: + auth = get_netrc_auth(request.url) + p = PreparedRequest() + p.prepare( + method=request.method.upper(), + url=request.url, + files=request.files, + data=request.data, + json=request.json, + headers=merge_setting( + request.headers, self.headers, dict_class=CaseInsensitiveDict + ), + params=merge_setting(request.params, self.params), + auth=merge_setting(auth, self.auth), + cookies=merged_cookies, + hooks=merge_hooks(request.hooks, self.hooks), + ) + return p + + def request( + self, + method, + url, + params=None, + data=None, + headers=None, + cookies=None, + files=None, + auth=None, + timeout=None, + allow_redirects=True, + proxies=None, + hooks=None, + stream=None, + verify=None, + cert=None, + json=None, + ): + """Constructs a :class:`Request `, prepares it, and sends it. + Returns :class:`Response ` object. + + :param method: method for the new :class:`Request` object. + :param url: URL for the new :class:`Request` object. + :param params: (optional) Dictionary or bytes to be sent in the query + string for the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. + :param json: (optional) json to send in the body of the + :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to send with the + :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the + :class:`Request`. + :param files: (optional) Dictionary of ``'filename': file-like-objects`` + for multipart encoding upload. + :param auth: (optional) Auth tuple or callable to enable + Basic/Digest/Custom HTTP Auth. + :param timeout: (optional) How long to wait for the server to send + data before giving up, as a float, or a :ref:`(connect timeout, + read timeout) ` tuple. + :type timeout: float or tuple + :param allow_redirects: (optional) Set to True by default. + :type allow_redirects: bool + :param proxies: (optional) Dictionary mapping protocol or protocol and + hostname to the URL of the proxy. + :param stream: (optional) whether to immediately download the response + content. Defaults to ``False``. + :param verify: (optional) Either a boolean, in which case it controls whether we verify + the server's TLS certificate, or a string, in which case it must be a path + to a CA bundle to use. Defaults to ``True``. + :param cert: (optional) if String, path to ssl client cert file (.pem). + If Tuple, ('cert', 'key') pair. + :rtype: requests.Response + """ + # Create the Request. + req = Request( + method=method.upper(), + url=url, + headers=headers, + files=files, + data=data or {}, + json=json, + params=params or {}, + auth=auth, + cookies=cookies, + hooks=hooks, + ) + prep = self.prepare_request(req) + proxies = proxies or {} + settings = self.merge_environment_settings( + prep.url, proxies, stream, verify, cert + ) + # Send the request. + send_kwargs = {"timeout": timeout, "allow_redirects": allow_redirects} + send_kwargs.update(settings) + resp = self.send(prep, **send_kwargs) + return resp + + def get(self, url, **kwargs): + r"""Sends a GET request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response + """ + kwargs.setdefault("allow_redirects", True) + return self.request("GET", url, **kwargs) + + def options(self, url, **kwargs): + r"""Sends a OPTIONS request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response + """ + kwargs.setdefault("allow_redirects", True) + return self.request("OPTIONS", url, **kwargs) + + def head(self, url, **kwargs): + r"""Sends a HEAD request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response + """ + kwargs.setdefault("allow_redirects", False) + return self.request("HEAD", url, **kwargs) + + def post(self, url, data=None, json=None, **kwargs): + r"""Sends a POST request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. + :param json: (optional) json to send in the body of the :class:`Request`. + :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response + """ + return self.request("POST", url, data=data, json=json, **kwargs) + + def put(self, url, data=None, **kwargs): + r"""Sends a PUT request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. + :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response + """ + return self.request("PUT", url, data=data, **kwargs) + + def patch(self, url, data=None, **kwargs): + r"""Sends a PATCH request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. + :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response + """ + return self.request("PATCH", url, data=data, **kwargs) + + def delete(self, url, **kwargs): + r"""Sends a DELETE request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response + """ + return self.request("DELETE", url, **kwargs) + + def send(self, request, **kwargs): + """Send a given PreparedRequest. + + :rtype: requests.Response + """ + # Set defaults that the hooks can utilize to ensure they always have + # the correct parameters to reproduce the previous request. + kwargs.setdefault("stream", self.stream) + kwargs.setdefault("verify", self.verify) + kwargs.setdefault("cert", self.cert) + kwargs.setdefault("proxies", self.proxies) + # It's possible that users might accidentally send a Request object. + # Guard against that specific failure case. + if isinstance(request, Request): + raise ValueError("You can only send PreparedRequests.") + + # Set up variables needed for resolve_redirects and dispatching of + # hooks + allow_redirects = kwargs.pop("allow_redirects", True) + stream = kwargs.get("stream") + hooks = request.hooks + # Get the appropriate adapter to use + adapter = self.get_adapter(url=request.url) + # Start time (approximately) of the request + start = preferred_clock() + # Send the request + r = adapter.send(request, **kwargs) + # Total elapsed time of the request (approximately) + elapsed = preferred_clock() - start + r.elapsed = timedelta(seconds=elapsed) + # Response manipulation hooks. + r = dispatch_hook("response", hooks, r, **kwargs) + # Persist cookies + if r.history: + # If the hooks create history then we want those cookies too + for resp in r.history: + extract_cookies_to_jar(self.cookies, resp.request, resp.raw) + extract_cookies_to_jar(self.cookies, request, r.raw) + # Redirect resolving generator. + gen = self.resolve_redirects(r, request, **kwargs) + # Resolve redirects, if allowed. + history = [resp for resp in gen] if allow_redirects else [] + # If there is a history, replace ``r`` with the last response + if history: + r = history.pop() + # If redirects aren't being followed, store the response on the Request for Response.next(). + if not allow_redirects: + try: + r._next = next( + self.resolve_redirects(r, request, yield_requests=True, **kwargs) + ) + except StopIteration: + pass + if not stream: + r.content + return r + + def merge_environment_settings(self, url, proxies, stream, verify, cert): + """ + Check the environment and merge it with some settings. + + :rtype: dict + """ + # Merge all the kwargs except for proxies. + stream = merge_setting(stream, self.stream) + verify = merge_setting(verify, self.verify) + cert = merge_setting(cert, self.cert) + # Gather clues from the surrounding environment. + # We do this after merging the Session values to make sure we don't + # accidentally exclude them. + if self.trust_env: + # Look for requests environment configuration and be compatible + # with cURL. + if verify is True or verify is None: + verify = ( + os.environ.get("REQUESTS_CA_BUNDLE") + or os.environ.get("CURL_CA_BUNDLE") + or verify + ) + # Now we handle proxies. + # Proxies need to be built up backwards. This is because None values + # can delete proxy information, which can then be re-added by a more + # specific layer. So we begin by getting the environment's proxies, + # then add the Session, then add the request. + no_proxy = proxies.get("no_proxy") if proxies is not None else None + if no_proxy is None: + no_proxy = self.proxies.get("no_proxy") + env_proxies = {} + if self.trust_env: + env_proxies = get_environ_proxies(url, no_proxy=no_proxy) or {} + new_proxies = merge_setting(self.proxies, env_proxies) + proxies = merge_setting(proxies, new_proxies) + return {"verify": verify, "proxies": proxies, "stream": stream, "cert": cert} + + def get_adapter(self, url): + """ + Returns the appropriate connection adapter for the given URL. + + :rtype: requests.adapters.BaseAdapter + """ + for (prefix, adapter) in self.adapters.items(): + if url.lower().startswith(prefix): + return adapter + + # Nothing matches :-/ + raise InvalidScheme("No connection adapters were found for '%s'" % url) + + def close(self): + """Closes all adapters and, as such, the Session.""" + for v in self.adapters.values(): + v.close() + + def mount(self, prefix, adapter): + """Registers a connection adapter to a prefix. + + Adapters are sorted in descending order by prefix length. + """ + self.adapters[prefix] = adapter + keys_to_move = [k for k in self.adapters if len(k) < len(prefix)] + for key in keys_to_move: + self.adapters[key] = self.adapters.pop(key) + + def __getstate__(self): + state = {attr: getattr(self, attr, None) for attr in self.__slots__} + return state + + def __setstate__(self, state): + for attr, value in state.items(): + setattr(self, attr, value) + + +class AsyncSession(Session): + """docstring for AsyncSession""" + + def __init__(self, backend=None): + self.backend = backend or TrioBackend() + super(AsyncSession, self).__init__() + self.mount("https://", AsyncHTTPAdapter(backend=self.backend)) + self.mount("http://", AsyncHTTPAdapter(backend=self.backend)) + + async def get(self, url, **kwargs): + r"""Sends a GET request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response + """ + kwargs.setdefault("allow_redirects", True) + return await self.request("GET", url, **kwargs) + + async def options(self, url, **kwargs): + r"""Sends a OPTIONS request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response + """ + kwargs.setdefault("allow_redirects", True) + return await self.request("OPTIONS", url, **kwargs) + + async def head(self, url, **kwargs): + r"""Sends a HEAD request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response + """ + kwargs.setdefault("allow_redirects", False) + return await self.request("HEAD", url, **kwargs) + + async def post(self, url, data=None, json=None, **kwargs): + r"""Sends a POST request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param json: (optional) json to send in the body of the :class:`Request`. + :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response + """ + return await self.request("POST", url, data=data, json=json, **kwargs) + + async def put(self, url, data=None, **kwargs): + r"""Sends a PUT request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response + """ + return await self.request("PUT", url, data=data, **kwargs) + + async def patch(self, url, data=None, **kwargs): + r"""Sends a PATCH request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response + """ + return await self.request("PATCH", url, data=data, **kwargs) + + async def delete(self, url, **kwargs): + r"""Sends a DELETE request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param \*\*kwargs: Optional arguments that ``request`` takes. + :rtype: requests.Response + """ + return await self.request("DELETE", url, **kwargs) + + async def request( + self, + method, + url, + params=None, + data=None, + headers=None, + cookies=None, + files=None, + auth=None, + timeout=None, + allow_redirects=True, + proxies=None, + hooks=None, + stream=None, + verify=None, + cert=None, + json=None, + ): + """Constructs a :class:`Request `, prepares it, and sends it. + Returns :class:`Response ` object. + + :param method: method for the new :class:`Request` object. + :param url: URL for the new :class:`Request` object. + :param params: (optional) Dictionary or bytes to be sent in the query + string for the :class:`Request`. + :param data: (optional) Dictionary, bytes, or file-like object to send + in the body of the :class:`Request`. + :param json: (optional) json to send in the body of the + :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to send with the + :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the + :class:`Request`. + :param files: (optional) Dictionary of ``'filename': file-like-objects`` + for multipart encoding upload. + :param auth: (optional) Auth tuple or callable to enable + Basic/Digest/Custom HTTP Auth. + :param timeout: (optional) How long to wait for the server to send + data before giving up, as a float, or a :ref:`(connect timeout, + read timeout) ` tuple. + :type timeout: float or tuple + :param allow_redirects: (optional) Set to True by default. + :type allow_redirects: bool + :param proxies: (optional) Dictionary mapping protocol or protocol and + hostname to the URL of the proxy. + :param stream: (optional) whether to immediately download the response + content. Defaults to ``False``. + :param verify: (optional) Either a boolean, in which case it controls whether we verify + the server's TLS certificate, or a string, in which case it must be a path + to a CA bundle to use. Defaults to ``True``. + :param cert: (optional) if String, path to ssl client cert file (.pem). + If Tuple, ('cert', 'key') pair. + :rtype: requests.Response + """ + # Create the Request. + req = Request( + method=method.upper(), + url=url, + headers=headers, + files=files, + data=data or {}, + json=json, + params=params or {}, + auth=auth, + cookies=cookies, + hooks=hooks, + ) + prep = self.prepare_request(req) + proxies = proxies or {} + settings = self.merge_environment_settings( + prep.url, proxies, stream, verify, cert + ) + # Send the request. + send_kwargs = {"timeout": timeout, "allow_redirects": allow_redirects} + send_kwargs.update(settings) + resp = await self.send(prep, **send_kwargs) + return resp + + async def send(self, request, **kwargs): + """Send a given PreparedRequest. + + :rtype: requests.Response + """ + # Set defaults that the hooks can utilize to ensure they always have + # the correct parameters to reproduce the previous request. + kwargs.setdefault("stream", self.stream) + kwargs.setdefault("verify", self.verify) + kwargs.setdefault("cert", self.cert) + kwargs.setdefault("proxies", self.proxies) + # It's possible that users might accidentally send a Request object. + # Guard against that specific failure case. + if isinstance(request, Request): + raise ValueError("You can only send PreparedRequests.") + + # Set up variables needed for resolve_redirects and dispatching of + # hooks + allow_redirects = kwargs.pop("allow_redirects", True) + stream = kwargs.get("stream") + hooks = request.hooks + # Get the appropriate adapter to use + adapter = self.get_adapter(url=request.url) + # Start time (approximately) of the request + start = preferred_clock() + # Send the request + r = await adapter.send(request, **kwargs) + # Total elapsed time of the request (approximately) + elapsed = preferred_clock() - start + r.elapsed = timedelta(seconds=elapsed) + # Response manipulation hooks. + r = dispatch_hook("response", hooks, r, **kwargs) + # Persist cookies + if r.history: + # If the hooks create history then we want those cookies too + for resp in r.history: + extract_cookies_to_jar(self.cookies, resp.request, resp.raw) + extract_cookies_to_jar(self.cookies, request, r.raw) + # Redirect resolving generator. + gen = self.resolve_redirects(r, request, **kwargs) + # Resolve redirects, if allowed. + history = [resp for resp in gen] if allow_redirects else [] + # If there is a history, replace ``r`` with the last response + if history: + r = history.pop() + # If redirects aren't being followed, store the response on the Request for Response.next(). + if not allow_redirects: + try: + r._next = next( + self.resolve_redirects(r, request, yield_requests=True, **kwargs) + ) + except StopIteration: + pass + if not stream: + await r.content + return r diff --git a/requests3/status_codes.py b/requests3/status_codes.py new file mode 100644 index 00000000..433ab081 --- /dev/null +++ b/requests3/status_codes.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +from .structures import LookupDict + +_codes = { + # Informational. + 100: ("continue",), + 101: ("switching_protocols",), + 102: ("processing",), + 103: ("checkpoint",), + 122: ("uri_too_long", "request_uri_too_long"), + 200: ("ok", "okay", "all_ok", "all_okay", "all_good", "\\o/", "✓"), + 201: ("created",), + 202: ("accepted",), + 203: ("non_authoritative_info", "non_authoritative_information"), + 204: ("no_content",), + 205: ("reset_content", "reset"), + 206: ("partial_content", "partial"), + 207: ("multi_status", "multiple_status", "multi_stati", "multiple_stati"), + 208: ("already_reported",), + 226: ("im_used",), + # Redirection. + 300: ("multiple_choices",), + 301: ("moved_permanently", "moved", "\\o-"), + 302: ("found",), + 303: ("see_other", "other"), + 304: ("not_modified",), + 305: ("use_proxy",), + 306: ("switch_proxy",), + 307: ("temporary_redirect", "temporary_moved", "temporary"), + 308: ("permanent_redirect", "resume_incomplete", "resume"), + # These 2 to be removed in 3.0 + # Client Error. + 400: ("bad_request", "bad"), + 401: ("unauthorized",), + 402: ("payment_required", "payment"), + 403: ("forbidden",), + 404: ("not_found", "-o-"), + 405: ("method_not_allowed", "not_allowed"), + 406: ("not_acceptable",), + 407: ("proxy_authentication_required", "proxy_auth", "proxy_authentication"), + 408: ("request_timeout", "timeout"), + 409: ("conflict",), + 410: ("gone",), + 411: ("length_required",), + 412: ("precondition_failed", "precondition"), + 413: ("request_entity_too_large",), + 414: ("request_uri_too_large",), + 415: ("unsupported_media_type", "unsupported_media", "media_type"), + 416: ( + "requested_range_not_satisfiable", + "requested_range", + "range_not_satisfiable", + ), + 417: ("expectation_failed",), + 418: ("im_a_teapot", "teapot", "i_am_a_teapot"), + 421: ("misdirected_request",), + 422: ("unprocessable_entity", "unprocessable"), + 423: ("locked",), + 424: ("failed_dependency", "dependency"), + 425: ("unordered_collection", "unordered"), + 426: ("upgrade_required", "upgrade"), + 428: ("precondition_required", "precondition"), + 429: ("too_many_requests", "too_many"), + 431: ("header_fields_too_large", "fields_too_large"), + 444: ("no_response", "none"), + 449: ("retry_with", "retry"), + 450: ("blocked_by_windows_parental_controls", "parental_controls"), + 451: ("unavailable_for_legal_reasons", "legal_reasons"), + 499: ("client_closed_request",), + # Server Error. + 500: ("internal_server_error", "server_error", "/o\\", "✗"), + 501: ("not_implemented",), + 502: ("bad_gateway",), + 503: ("service_unavailable", "unavailable"), + 504: ("gateway_timeout",), + 505: ("http_version_not_supported", "http_version"), + 506: ("variant_also_negotiates",), + 507: ("insufficient_storage",), + 509: ("bandwidth_limit_exceeded", "bandwidth"), + 510: ("not_extended",), + 511: ("network_authentication_required", "network_auth", "network_authentication"), +} +codes = LookupDict(name="status_codes") +for code, titles in _codes.items(): + for title in titles: # type: ignore + setattr(codes, title, code) + if not title.startswith(("\\", "/")): + setattr(codes, title.upper(), code) diff --git a/requests3/structures.py b/requests3/structures.py new file mode 100644 index 00000000..c88d744d --- /dev/null +++ b/requests3/structures.py @@ -0,0 +1,229 @@ +# -*- coding: utf-8 -*- +""" +requests.structures +~~~~~~~~~~~~~~~~~~~ + +Data structures that power Requests. +""" + +import collections + +from .basics import basestring, OrderedDict + + +class CaseInsensitiveDict(collections.MutableMapping): + """A case-insensitive ``dict``-like object. + + Implements all methods and operations of + ``collections.MutableMapping`` as well as dict's ``copy``. Also + provides ``lower_items``. + + All keys are expected to be strings. The structure remembers the + case of the last key to be set, and ``iter(instance)``, + ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()`` + will contain case-sensitive keys. However, querying and contains + testing is case insensitive:: + + cid = CaseInsensitiveDict() + cid['Accept'] = 'application/json' + cid['aCCEPT'] == 'application/json' # True + list(cid) == ['Accept'] # True + + For example, ``headers['content-encoding']`` will return the + value of a ``'Content-Encoding'`` response header, regardless + of how the header name was originally stored. + + If the constructor, ``.update``, or equality comparison + operations are given keys that have equal ``.lower()``s, the + behavior is undefined. + """ + __slots__ = ('_store') + + def __init__(self, data=None, **kwargs): + self._store = collections.OrderedDict() + if data is None: + data = {} + self.update(data, **kwargs) + + def __setitem__(self, key, value): + # Use the lowercased key for lookups, but store the actual + # key alongside the value. + self._store[key.lower()] = (key, value) + + def __getitem__(self, key): + return self._store[key.lower()][1] + + def __delitem__(self, key): + del self._store[key.lower()] + + def __iter__(self): + return (casedkey for casedkey, mappedvalue in self._store.values()) + + def __len__(self): + return len(self._store) + + def lower_items(self): + """Like iteritems(), but with all lowercase keys.""" + return ( + (lowerkey, keyval[1]) for (lowerkey, keyval) in self._store.items() + ) + + def __eq__(self, other): + if isinstance(other, collections.Mapping): + other = CaseInsensitiveDict(other) + else: + return NotImplemented + + # Compare insensitively + return dict(self.lower_items()) == dict(other.lower_items()) + + + # Copy is required + def copy(self): + return CaseInsensitiveDict(self._store.values()) + + def __repr__(self): + return str(dict(self.items())) + + +class HTTPHeaderDict(CaseInsensitiveDict): + """A case-insensitive ``dict``-like object suitable for HTTP headers that + supports multiple values with the same key, via the ``add``, ``extend``, + ``multiget`` and ``multiset`` methods. + """ + + def __init__(self, data=None, **kwargs): + super(HTTPHeaderDict, self).__init__() + self.extend({} if data is None else data, **kwargs) + + + # We'll store tuples in the internal dictionary, but present them as a + # concatenated string when we use item access methods. + # + def __setitem__(self, key, val): + # Special–case null values. + if (not isinstance(val, basestring)) and (val is not None): + raise ValueError('only string-type values (or None) are allowed') + + super(HTTPHeaderDict, self).__setitem__(key, (val,)) + + def __getitem__(self, key): + val = super(HTTPHeaderDict, self).__getitem__(key) + # Special–case null values. + if len(val) == 1 and val[0] is None: + return val[0] + + return ', '.join(val) + + def lower_items(self): + return ( + (lk, ', '.join(vals)) for (lk, (k, vals)) in self._store.items() + ) + + def copy(self): + return type(self)(self) + + def getlist(self, key): + """Returns a list of all the values for the named field. Returns an + empty list if the key isn't present in the dictionary.""" + return list(self._store.get(key.lower(), (None, []))[1]) + + def setlist(self, key, values): + """Set a sequence of strings to the associated key - this will overwrite + any previously stored value.""" + if not isinstance(values, (list, tuple)): + raise ValueError('argument is not sequence') + + if any(not isinstance(v, basestring) for v in values): + raise ValueError('non-string items in sequence') + + if not values: + self.pop(key, None) + return + + super(HTTPHeaderDict, self).__setitem__(key, tuple(values)) + + def _extend(self, key, values): + new_value_tpl = key, values + # Inspired by urllib3's implementation - use one call which should be + # suitable for the common case. + old_value_tpl = self._store.setdefault(key.lower(), new_value_tpl) + if old_value_tpl is not new_value_tpl: + old_key, old_values = old_value_tpl + self._store[key.lower()] = (old_key, old_values + values) + + def add(self, key, val): + """Adds a key, value pair to this dictionary - if there is already a + value for this key, then the value will be appended to those values. + """ + if not isinstance(val, basestring): + raise ValueError('value must be a string-type object') + + self._extend(key, (val,)) + + def extend(self, *args, **kwargs): + """Like update, but will add values to existing sequences rather than + replacing them. You can pass a mapping object or a sequence of two + tuples - values in these objects can be strings or sequence of strings. + """ + if len(args) > 1: + raise TypeError( + f"extend() takes at most 1 positional " + "arguments ({len(args)} given)" + ) + + for other in args + (kwargs,): + if isinstance(other, collections.Mapping): + # See if looks like a HTTPHeaderDict (either urllib3's + # implementation or ours). If so, then we have to add values + # in one go for each key. + multiget = getattr(other, 'getlist', None) + if multiget: + for key in other: + self._extend(key, tuple(multiget(key))) + continue + + # Otherwise, just walk over items to get them. + item_seq = other.items() + else: + item_seq = other + for ik, iv in item_seq: + if isinstance(iv, basestring): + self._extend(ik, (iv,)) + elif any(not isinstance(v, basestring) for v in iv): + raise ValueError('non-string items in sequence') + + else: + self._extend(ik, tuple(iv)) + + @property + def _as_dict(self): + """A dictionary representation of the HTTPHeaderDict.""" + d = {} + for k, vals in self._store.values(): + d[k] = vals[0] if len(vals) == 1 else vals + return d + + def __repr__(self): + return repr(self._as_dict) + + +class LookupDict(dict): + """Dictionary lookup object.""" + + def __init__(self, name=None): + self.name = name + super(LookupDict, self).__init__() + + def __repr__(self): + return f'' + + def __getitem__(self, key): + # We allow fall-through here, so values default to None + return self.__dict__.get(key, None) + + def __iter__(self): + return super(LookupDict, self).__dir__() + + def get(self, key, default=None): + return self.__dict__.get(key, default) diff --git a/requests3/types.py b/requests3/types.py new file mode 100644 index 00000000..4b609867 --- /dev/null +++ b/requests3/types.py @@ -0,0 +1,70 @@ +from typing import ( + Callable, + Optional, + Union, + Any, + Iterable, + List, + Mapping, + MutableMapping, + Tuple, + IO, + Text, + Type, + Dict, +) + +from .import auth +from .models import Response, PreparedRequest +from .cookies import RequestsCookieJar +from .sessions import Session + +_ParamsMappingValueType = Union[ + str, bytes, int, float, Iterable[Union[str, bytes, int, float]] +] +Params = Optional[ + Union[ + Mapping[Union[str, bytes, int, float], _ParamsMappingValueType], + Union[str, bytes], + Tuple[Union[str, bytes, int, float], _ParamsMappingValueType], + Mapping[str, _ParamsMappingValueType], + Mapping[bytes, _ParamsMappingValueType], + Mapping[int, _ParamsMappingValueType], + Mapping[float, _ParamsMappingValueType], + ] +] +Data = Union[ + None, + bytes, + MutableMapping[str, str], + MutableMapping[str, Text], + MutableMapping[Text, str], + MutableMapping[Text, Text], + Iterable[Tuple[str, str]], + IO, +] +_Hook = Callable[[Response], Any] +Method = str +URL = str +Headers = Optional[Union[None, MutableMapping[Text, Text]]] +Cookies = Optional[Union[None, RequestsCookieJar, MutableMapping[Text, Text]]] +Files = Optional[MutableMapping[Text, IO]] +Auth = Union[ + None, + Tuple[Text, Text], + auth.AuthBase, + Callable[[PreparedRequest], PreparedRequest], +] +Timeout = Union[None, float, Tuple[float, float]] +AllowRedirects = Optional[bool] +Proxies = Optional[MutableMapping[Text, Text]] +Hooks = Optional[MutableMapping[Text, Union[Iterable[_Hook], _Hook]]] +Stream = Optional[bool] +Verify = Union[None, bool, Text] +Cert = Union[Text, Tuple[Text, Text]] +JSON = Optional[MutableMapping] +Help = Dict +Host = str +Sequence = List +Filename = str +KeyValueList = List[Tuple[Text, Text]] diff --git a/requests/utils.py b/requests3/utils.py similarity index 64% rename from requests/utils.py rename to requests3/utils.py index 8170a8d2..445e7b90 100644 --- a/requests/utils.py +++ b/requests3/utils.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ requests.utils ~~~~~~~~~~~~~~ @@ -18,72 +17,76 @@ import struct import sys import tempfile import warnings -import zipfile +import typing from .__version__ import __version__ from . import certs -# to_native_string is unused here, but imported here for backwards compatibility -from ._internal_utils import to_native_string -from .compat import parse_http_list as _parse_list_header -from .compat import ( - quote, urlparse, bytes, str, OrderedDict, unquote, getproxies, - proxy_bypass, urlunparse, basestring, integer_types, is_py3, - proxy_bypass_environment, getproxies_environment, Mapping) + +from .basics import parse_http_list as _parse_list_header +from .basics import ( + quote, + urlparse, + bytes, + str, + unquote, + getproxies, + proxy_bypass, + urlunparse, + basestring, + integer_types, + proxy_bypass_environment, + getproxies_environment, +) from .cookies import cookiejar_from_dict -from .structures import CaseInsensitiveDict +from .structures import HTTPHeaderDict +from .cookies import RequestsCookieJar from .exceptions import ( - InvalidURL, InvalidHeader, FileModeWarning, UnrewindableBodyError) - -NETRC_FILES = ('.netrc', '_netrc') + InvalidURL, + InvalidHeader, + FileModeWarning, + UnrewindableBodyError, +) +NETRC_FILES = (".netrc", "_netrc") DEFAULT_CA_BUNDLE_PATH = certs.where() +if platform.system() == "Windows": -DEFAULT_PORTS = {'http': 80, 'https': 443} - - -if sys.platform == 'win32': # provide a proxy_bypass version on Windows without DNS lookups - - def proxy_bypass_registry(host): - try: - if is_py3: - import winreg - else: - import _winreg as winreg - except ImportError: - return False + def proxy_bypass_registry(host: str) -> bool: + import winreg # typing: ignore try: - internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, - r'Software\Microsoft\Windows\CurrentVersion\Internet Settings') - # ProxyEnable could be REG_SZ or REG_DWORD, normalizing it - proxyEnable = int(winreg.QueryValueEx(internetSettings, - 'ProxyEnable')[0]) - # ProxyOverride is almost always a string - proxyOverride = winreg.QueryValueEx(internetSettings, - 'ProxyOverride')[0] + internetSettings = winreg.OpenKey( + winreg.HKEY_CURRENT_USER, + r"Software\Microsoft\Windows\CurrentVersion\Internet Settings", + ) + proxyEnable = winreg.QueryValueEx(internetSettings, "ProxyEnable")[0] + proxyOverride = winreg.QueryValueEx(internetSettings, "ProxyOverride")[0] except OSError: return False + if not proxyEnable or not proxyOverride: return False # make a check value list from the registry entry: replace the # '' string by the localhost entry and the corresponding # canonical entry. - proxyOverride = proxyOverride.split(';') + proxyOverride = proxyOverride.split(";") # now check if we match one of the registry values. for test in proxyOverride: - if test == '': - if '.' not in host: + if test == "": + if "." not in host: return True - test = test.replace(".", r"\.") # mask dots - test = test.replace("*", r".*") # change glob sequence - test = test.replace("?", r".") # change glob char + + test = test.replace(".", r"\.") # mask dots + test = test.replace("*", r".*") # change glob sequence + test = test.replace("?", r".") # change glob char if re.match(test, host, re.I): return True + return False - def proxy_bypass(host): # noqa + def proxy_bypass(host: str) -> bool: # noqa """Return True, if the host should be bypassed. Checks proxy settings gathered from the environment, if specified, @@ -91,51 +94,50 @@ if sys.platform == 'win32': """ if getproxies_environment(): return proxy_bypass_environment(host) + else: return proxy_bypass_registry(host) -def dict_to_sequence(d): +def dict_to_sequence( + d: dict +) -> typing.Union[typing.Optional[typing.ItemsView[typing.Any, typing.Any]], dict]: """Returns an internal sequence dictionary update.""" - - if hasattr(d, 'items'): - d = d.items() + if hasattr(d, "items"): + return d.items() return d -def super_len(o): +def super_len(o) -> int: total_length = None current_position = 0 - - if hasattr(o, '__len__'): + if hasattr(o, "__len__"): total_length = len(o) - - elif hasattr(o, 'len'): + elif hasattr(o, "len"): total_length = o.len - - elif hasattr(o, 'fileno'): + elif hasattr(o, "fileno"): try: fileno = o.fileno() except io.UnsupportedOperation: pass else: total_length = os.fstat(fileno).st_size - # Having used fstat to determine the file length, we need to # confirm that this file was opened up in binary mode. - if 'b' not in o.mode: - warnings.warn(( - "Requests has determined the content-length for this " - "request using the binary size of the file: however, the " - "file has been opened in text mode (i.e. without the 'b' " - "flag in the mode). This may lead to an incorrect " - "content-length. In Requests 3.0, support will be removed " - "for files in text mode."), - FileModeWarning + if "b" not in o.mode: + warnings.warn( + ( + "Requests has determined the content-length for this " + "request using the binary size of the file: however, the " + "file has been opened in typing.Text mode (i.e. without the 'b' " + "flag in the mode). This may lead to an incorrect " + "content-length. In Requests 3.0, support will be removed " + "for files in typing.Text mode." + ), + FileModeWarning, ) - - if hasattr(o, 'tell'): + if hasattr(o, "tell"): try: current_position = o.tell() except (OSError, IOError): @@ -146,41 +148,38 @@ def super_len(o): if total_length is not None: current_position = total_length else: - if hasattr(o, 'seek') and total_length is None: + if hasattr(o, "seek") and total_length is None: # StringIO and BytesIO have seek but no useable fileno try: # seek to end of file o.seek(0, 2) total_length = o.tell() - # seek back to current position to support # partially read file-like objects o.seek(current_position or 0) except (OSError, IOError): total_length = 0 - if total_length is None: total_length = 0 - return max(0, total_length - current_position) -def get_netrc_auth(url, raise_errors=False): +def get_netrc_auth( + url: str, raise_errors: bool = False +) -> typing.Optional[typing.Tuple[typing.Text, typing.Text]]: """Returns the Requests tuple auth for a given url from netrc.""" - try: from netrc import netrc, NetrcParseError netrc_path = None - for f in NETRC_FILES: try: - loc = os.path.expanduser('~/{}'.format(f)) + loc = os.path.expanduser(f"~/{f}") except KeyError: # os.path.expanduser can fail when $HOME is undefined and # getpwuid fails. See https://bugs.python.org/issue20164 & # https://github.com/requests/requests/issues/1846 - return + return None if os.path.exists(loc): netrc_path = loc @@ -188,23 +187,17 @@ def get_netrc_auth(url, raise_errors=False): # Abort early if there isn't one. if netrc_path is None: - return + return None ri = urlparse(url) - - # Strip port numbers from netloc. This weird `if...encode`` dance is - # used for Python 3.2, which doesn't support unicode literals. - splitstr = b':' - if isinstance(url, str): - splitstr = splitstr.decode('ascii') - host = ri.netloc.split(splitstr)[0] - + host = ri.netloc.split(":")[0] try: _netrc = netrc(netrc_path).authenticators(host) if _netrc: # Return with login / password - login_i = (0 if _netrc[0] else 1) + login_i = 0 if _netrc[0] else 1 return (_netrc[login_i], _netrc[2]) + except (NetrcParseError, IOError): # If there was a parsing error or a permissions issue reading the file, # we'll just skip netrc auth unless explicitly asked to raise errors. @@ -214,13 +207,13 @@ def get_netrc_auth(url, raise_errors=False): # AppEngine hackiness. except (ImportError, AttributeError): pass + return None -def guess_filename(obj): +def guess_filename(obj) -> str: """Tries to guess the filename of the given object.""" - name = getattr(obj, 'name', None) - if (name and isinstance(name, basestring) and name[0] != '<' and - name[-1] != '>'): + name = getattr(obj, "name", None) + if name and isinstance(name, basestring) and name[0] != "<" and name[-1] != ">": return os.path.basename(name) @@ -238,7 +231,7 @@ def extract_zipped_paths(path): archive, member = os.path.split(path) while archive and not os.path.exists(archive): archive, prefix = os.path.split(archive) - member = '/'.join([prefix, member]) + member = "/".join([prefix, member]) if not zipfile.is_zipfile(archive): return path @@ -249,7 +242,7 @@ def extract_zipped_paths(path): # we have a valid zip archive and a valid member of that archive tmp = tempfile.gettempdir() - extracted_path = os.path.join(tmp, *member.split('/')) + extracted_path = os.path.join(tmp, *member.split("/")) if not os.path.exists(extracted_path): extracted_path = zip_file.extract(member, path=tmp) @@ -276,12 +269,12 @@ def from_key_val_list(value): return None if isinstance(value, (str, bytes, bool, int)): - raise ValueError('cannot encode objects that are not 2-tuples') + raise ValueError("cannot encode objects that are not 2-tuples") - return OrderedDict(value) + return collections.OrderedDict(value) -def to_key_val_list(value): +def to_key_val_list(value) -> typing.List[typing.Tuple[typing.Text, typing.Text]]: """Take an object and test to see if it can be represented as a dictionary. If it can be, return a list of tuples, e.g., @@ -300,16 +293,15 @@ def to_key_val_list(value): return None if isinstance(value, (str, bytes, bool, int)): - raise ValueError('cannot encode objects that are not 2-tuples') + raise ValueError("cannot encode objects that are not 2-tuples") if isinstance(value, Mapping): value = value.items() - return list(value) # From mitsuhiko/werkzeug (used with permission). -def parse_list_header(value): +def parse_list_header(value: str) -> typing.List[typing.Text]: """Parse lists as described by RFC 2068 Section 2. In particular, parse comma-separated lists where the elements of @@ -341,7 +333,7 @@ def parse_list_header(value): # From mitsuhiko/werkzeug (used with permission). -def parse_dict_header(value): +def parse_dict_header(value) -> dict: """Parse lists of key, value pairs as described by RFC 2068 Section 2 and convert them into a python dict: @@ -363,12 +355,13 @@ def parse_dict_header(value): :return: :class:`dict` :rtype: dict """ - result = {} + result = {} # type: dict for item in _parse_list_header(value): - if '=' not in item: + if "=" not in item: result[item] = None continue - name, value = item.split('=', 1) + + name, value = item.split("=", 1) if value[:1] == value[-1:] == '"': value = unquote_header_value(value[1:-1]) result[name] = value @@ -376,7 +369,7 @@ def parse_dict_header(value): # From mitsuhiko/werkzeug (used with permission). -def unquote_header_value(value, is_filename=False): +def unquote_header_value(value: str, is_filename: bool = False): r"""Unquotes a header value. (Reversal of :func:`quote_header_value`). This does not use the real unquoting but what browsers are actually using for quoting. @@ -390,123 +383,91 @@ def unquote_header_value(value, is_filename=False): # probably some other browsers as well. IE for example is # uploading files with "C:\foo\bar.txt" as filename value = value[1:-1] - # if this is a filename and the starting characters look like # a UNC path, then just return the value without quotes. Using the # replace sequence below on a UNC path has the effect of turning # the leading double slash into a single slash and then # _fix_ie_filename() doesn't work correctly. See #458. - if not is_filename or value[:2] != '\\\\': - return value.replace('\\\\', '\\').replace('\\"', '"') + if not is_filename or value[:2] != "\\\\": + return value.replace("\\\\", "\\").replace('\\"', '"') + return value -def dict_from_cookiejar(cj): +def dict_from_cookiejar(cj: RequestsCookieJar) -> dict: """Returns a key/value dictionary from a CookieJar. :param cj: CookieJar object to extract cookies from. :rtype: dict """ - cookie_dict = {} - for cookie in cj: cookie_dict[cookie.name] = cookie.value - return cookie_dict -def add_dict_to_cookiejar(cj, cookie_dict): +def add_dict_to_cookiejar( + cj: RequestsCookieJar, cookie_dict: dict +) -> RequestsCookieJar: """Returns a CookieJar from a key/value dictionary. :param cj: CookieJar to insert cookies into. :param cookie_dict: Dict of key/values to insert into CookieJar. :rtype: CookieJar """ - return cookiejar_from_dict(cookie_dict, cj) -def get_encodings_from_content(content): +def get_encodings_from_content(content: str) -> typing.List[str]: """Returns encodings from given content string. :param content: bytestring to extract encodings from. """ - warnings.warn(( - 'In requests 3.0, get_encodings_from_content will be removed. For ' - 'more information, please see the discussion on issue #2266. (This' - ' warning should only appear once.)'), - DeprecationWarning) - + warnings.warn( + ( + "In requests 3.0, get_encodings_from_content will be removed. For " + "more information, please see the discussion on issue #2266. (This" + " warning should only appear once.)" + ), + DeprecationWarning, + ) charset_re = re.compile(r']', flags=re.I) pragma_re = re.compile(r']', flags=re.I) xml_re = re.compile(r'^<\?xml.*?encoding=["\']*(.+?)["\'>]') - - return (charset_re.findall(content) + - pragma_re.findall(content) + - xml_re.findall(content)) + return ( + charset_re.findall(content) + + pragma_re.findall(content) + + xml_re.findall(content) + ) -def _parse_content_type_header(header): - """Returns content type and parameters from given header - - :param header: string - :return: tuple containing content type and dictionary of - parameters - """ - - tokens = header.split(';') - content_type, params = tokens[0].strip(), tokens[1:] - params_dict = {} - items_to_strip = "\"' " - - for param in params: - param = param.strip() - if param: - key, value = param, True - index_of_equals = param.find("=") - if index_of_equals != -1: - key = param[:index_of_equals].strip(items_to_strip) - value = param[index_of_equals + 1:].strip(items_to_strip) - params_dict[key.lower()] = value - return content_type, params_dict - - -def get_encoding_from_headers(headers): +def get_encoding_from_headers(headers: typing.MutableMapping) -> str: """Returns encodings from given HTTP Header Dict. :param headers: dictionary to extract encoding from. :rtype: str """ - - content_type = headers.get('content-type') - + content_type = headers.get("Content-Type") if not content_type: return None - content_type, params = _parse_content_type_header(content_type) + content_type, params = cgi.parse_header(content_type) + if "charset" in params: + return params["charset"].strip("'\"") - if 'charset' in params: - return params['charset'].strip("'\"") - - if 'text' in content_type: - return 'ISO-8859-1' + if "text" in content_type: + return "ISO-8859-1" def stream_decode_response_unicode(iterator, r): """Stream decodes a iterator.""" - - if r.encoding is None: - for item in iterator: - yield item - return - - decoder = codecs.getincrementaldecoder(r.encoding)(errors='replace') + decoder = codecs.getincrementaldecoder(r.encoding)(errors="replace") for chunk in iterator: rv = decoder.decode(chunk) if rv: yield rv - rv = decoder.decode(b'', final=True) + + rv = decoder.decode(b"", final=True) if rv: yield rv @@ -517,58 +478,38 @@ def iter_slices(string, slice_length): if slice_length is None or slice_length <= 0: slice_length = len(string) while pos < len(string): - yield string[pos:pos + slice_length] + yield string[pos : pos + slice_length] + pos += slice_length -def get_unicode_from_response(r): - """Returns the requested content back in unicode. - - :param r: Response object to get unicode content from. - - Tried: - - 1. charset from content-type - 2. fall back and replace all unicode characters - - :rtype: str - """ - warnings.warn(( - 'In requests 3.0, get_unicode_from_response will be removed. For ' - 'more information, please see the discussion on issue #2266. (This' - ' warning should only appear once.)'), - DeprecationWarning) - - tried_encodings = [] - - # Try charset from content-type - encoding = get_encoding_from_headers(r.headers) - - if encoding: - try: - return str(r.content, encoding) - except UnicodeError: - tried_encodings.append(encoding) - - # Fall back: - try: - return str(r.content, encoding, errors='replace') - except TypeError: - return r.content - - # The unreserved URI characters (RFC 3986) UNRESERVED_SET = frozenset( - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789-._~") + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789-._~" +) -def unquote_unreserved(uri): +def unquote_unreserved(uri: str) -> str: """Un-escape any percent-escape sequences in a URI that are unreserved characters. This leaves all reserved, illegal and non-ASCII bytes encoded. :rtype: str """ - parts = uri.split('%') + + # This convert function is used to optionally convert the output of `chr`. + # In Python 3, `chr` returns a unicode string, while in Python 2 it returns + # a bytestring. Here we deal with that by optionally converting. + def convert(is_bytes, c): + if is_bytes: + return c.encode("ascii") + + else: + return c + + # Handle both bytestrings and unicode strings. + splitchar = "%" + base = "" + parts = uri.split(splitchar) for i in range(1, len(parts)): h = parts[i][0:2] if len(h) == 2 and h.isalnum(): @@ -578,15 +519,15 @@ def unquote_unreserved(uri): raise InvalidURL("Invalid percent-escape sequence: '%s'" % h) if c in UNRESERVED_SET: - parts[i] = c + parts[i][2:] + parts[i] = convert(is_bytes=False, c=c) + parts[i][2:] else: - parts[i] = '%' + parts[i] + parts[i] = splitchar + parts[i] else: - parts[i] = '%' + parts[i] - return ''.join(parts) + parts[i] = splitchar + parts[i] + return base.join(parts) -def requote_uri(uri): +def requote_uri(uri: str) -> str: """Re-quote the given URI. This function passes the given URI through an unquote/quote cycle to @@ -601,6 +542,7 @@ def requote_uri(uri): # Then quote only illegal characters (do not quote reserved, # unreserved, or '%') return quote(unquote_unreserved(uri), safe=safe_with_percent) + except InvalidURL: # We couldn't unquote the given URI, so let's try quoting it, but # there may be unquoted '%'s in the URI. We need to make sure they're @@ -608,7 +550,7 @@ def requote_uri(uri): return quote(uri, safe=safe_without_percent) -def address_in_network(ip, net): +def address_in_network(ip: str, net: str) -> bool: """This function allows you to check if an IP belongs to a network subnet Example: returns True if ip = 192.168.1.1 and net = 192.168.1.0/24 @@ -616,25 +558,25 @@ def address_in_network(ip, net): :rtype: bool """ - ipaddr = struct.unpack('=L', socket.inet_aton(ip))[0] - netaddr, bits = net.split('/') - netmask = struct.unpack('=L', socket.inet_aton(dotted_netmask(int(bits))))[0] - network = struct.unpack('=L', socket.inet_aton(netaddr))[0] & netmask + ipaddr = struct.unpack("=L", socket.inet_aton(ip))[0] + netaddr, bits = net.split("/") + netmask = struct.unpack("=L", socket.inet_aton(dotted_netmask(int(bits))))[0] + network = struct.unpack("=L", socket.inet_aton(netaddr))[0] & netmask return (ipaddr & netmask) == (network & netmask) -def dotted_netmask(mask): +def dotted_netmask(mask: str) -> str: """Converts mask from /xx format to xxx.xxx.xxx.xxx Example: if mask is 24 function returns 255.255.255.0 :rtype: str """ - bits = 0xffffffff ^ (1 << 32 - mask) - 1 - return socket.inet_ntoa(struct.pack('>I', bits)) + bits = 0xFFFFFFFF ^ (1 << 32 - mask) - 1 + return socket.inet_ntoa(struct.pack(">I", bits)) -def is_ipv4_address(string_ip): +def is_ipv4_address(string_ip: str) -> bool: """ :rtype: bool """ @@ -642,18 +584,19 @@ def is_ipv4_address(string_ip): socket.inet_aton(string_ip) except socket.error: return False + return True -def is_valid_cidr(string_network): +def is_valid_cidr(string_network: str) -> bool: """ Very simple check of the cidr format in no_proxy variable. :rtype: bool """ - if string_network.count('/') == 1: + if string_network.count("/") == 1: try: - mask = int(string_network.split('/')[1]) + mask = int(string_network.split("/")[1]) except ValueError: return False @@ -661,16 +604,18 @@ def is_valid_cidr(string_network): return False try: - socket.inet_aton(string_network.split('/')[0]) + socket.inet_aton(string_network.split("/")[0]) except socket.error: return False + else: return False + return True @contextlib.contextmanager -def set_environ(env_name, value): +def set_environ(env_name: str, value: typing.Optional[str]) -> typing.Generator: """Set the environment variable 'env_name' to 'value' Save previous value, yield, and then restore the previous value stored in @@ -683,6 +628,7 @@ def set_environ(env_name, value): os.environ[env_name] = value try: yield + finally: if value_changed: if old_value is None: @@ -691,7 +637,7 @@ def set_environ(env_name, value): os.environ[env_name] = old_value -def should_bypass_proxies(url, no_proxy): +def should_bypass_proxies(url: str, no_proxy: typing.Optional[str]) -> bool: """ Returns whether we should bypass proxies or not. @@ -700,59 +646,44 @@ def should_bypass_proxies(url, no_proxy): # Prioritize lowercase environment variables over uppercase # to keep a consistent behaviour with other http projects (curl, wget). get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper()) - # First check whether no_proxy is defined. If it is, check that the URL # we're getting isn't in the no_proxy list. no_proxy_arg = no_proxy if no_proxy is None: - no_proxy = get_proxy('no_proxy') - parsed = urlparse(url) - - if parsed.hostname is None: - # URLs don't always have hostnames, e.g. file:/// urls. - return True - + no_proxy = get_proxy("no_proxy") + netloc = urlparse(url).netloc if no_proxy: # We need to check whether we match here. We need to see if we match # the end of the hostname, both with and without the port. - no_proxy = ( - host for host in no_proxy.replace(' ', '').split(',') if host - ) - - if is_ipv4_address(parsed.hostname): + no_proxy = (host for host in no_proxy.replace(" ", "").split(",") if host) + ip = netloc.split(":")[0] + if is_ipv4_address(ip): for proxy_ip in no_proxy: if is_valid_cidr(proxy_ip): if address_in_network(parsed.hostname, proxy_ip): return True - elif parsed.hostname == proxy_ip: + + elif ip == proxy_ip: # If no_proxy ip was defined in plain IP notation instead of cidr notation & # matches the IP of the index return True + else: host_with_port = parsed.hostname if parsed.port: - host_with_port += ':{}'.format(parsed.port) + host_with_port += ":{}".format(parsed.port) for host in no_proxy: - if parsed.hostname.endswith(host) or host_with_port.endswith(host): + if netloc.endswith(host) or netloc.split(":")[0].endswith(host): # The URL does match something in no_proxy, so we don't want # to apply the proxies on this URL. return True - with set_environ('no_proxy', no_proxy_arg): - # parsed.hostname can be `None` in cases such as a file URI. - try: - bypass = proxy_bypass(parsed.hostname) - except (TypeError, socket.gaierror): - bypass = False - - if bypass: - return True - - return False + with set_environ("no_proxy", no_proxy_arg): + return bool(proxy_bypass(netloc)) -def get_environ_proxies(url, no_proxy=None): +def get_environ_proxies(url: str, no_proxy: typing.Optional[bool] = None) -> dict: """ Return a dict of environment proxies. @@ -760,11 +691,14 @@ def get_environ_proxies(url, no_proxy=None): """ if should_bypass_proxies(url, no_proxy=no_proxy): return {} + else: return getproxies() -def select_proxy(url, proxies): +def select_proxy( + url: str, proxies: typing.Optional[typing.MutableMapping[typing.Text, typing.Text]] +): """Select a proxy for the url, if applicable. :param url: The url being for the request @@ -773,13 +707,13 @@ def select_proxy(url, proxies): proxies = proxies or {} urlparts = urlparse(url) if urlparts.hostname is None: - return proxies.get(urlparts.scheme, proxies.get('all')) + return proxies.get(urlparts.scheme, proxies.get("all")) proxy_keys = [ - urlparts.scheme + '://' + urlparts.hostname, + urlparts.scheme + "://" + urlparts.hostname, urlparts.scheme, - 'all://' + urlparts.hostname, - 'all', + "all://" + urlparts.hostname, + "all", ] proxy = None for proxy_key in proxy_keys: @@ -790,71 +724,80 @@ def select_proxy(url, proxies): return proxy -def default_user_agent(name="python-requests"): +def default_user_agent(name: str = "python-requests") -> str: """ Return a string representing the default user agent. :rtype: str """ - return '%s/%s' % (name, __version__) + return "%s/%s" % (name, __version__) -def default_headers(): +def default_headers() -> HTTPHeaderDict: """ - :rtype: requests.structures.CaseInsensitiveDict + :rtype: requests.structures.HTTPHeaderDict """ - return CaseInsensitiveDict({ - 'User-Agent': default_user_agent(), - 'Accept-Encoding': ', '.join(('gzip', 'deflate')), - 'Accept': '*/*', - 'Connection': 'keep-alive', - }) + return HTTPHeaderDict( + { + "User-Agent": default_user_agent(), + "Accept-Encoding": ", ".join(("gzip", "deflate")), + "Accept": "*/*", + "Connection": "keep-alive", + } + ) -def parse_header_links(value): +def parse_header_links(value: str) -> typing.List[typing.MutableMapping]: """Return a list of parsed link headers proxies. i.e. Link: ; rel=front; type="image/jpeg",; rel=back;type="image/jpeg" :rtype: list """ - - links = [] - - replace_chars = ' \'"' - + links = [] # type: typing.List + replace_chars = " '\"" value = value.strip(replace_chars) if not value: return links - for val in re.split(', *<', value): + for val in re.split(", *<", value): try: - url, params = val.split(';', 1) + url, params = val.split(";", 1) except ValueError: - url, params = val, '' - - link = {'url': url.strip('<> \'"')} - - for param in params.split(';'): + url, params = val, "" + link = {"url": url.strip("<> '\"")} + for param in params.split(";"): try: - key, value = param.split('=') + key, value = param.split("=") except ValueError: break link[key.strip(replace_chars)] = value.strip(replace_chars) - links.append(link) - return links +def is_valid_location(response) -> bool: + """Verify that multiple Location headers weren't + returned from the last response. + """ + headers = getattr(response.raw, "headers", None) + if headers is not None: + getlist = getattr(headers, "getlist", None) + if getlist is not None: + return len(getlist("location")) <= 1 + + # If response.raw isn't urllib3-like we can't reliably check this + return True + + # Null bytes; no need to recreate these on each call to guess_json_utf -_null = '\x00'.encode('ascii') # encoding to ASCII for Python 3 +_null = "\x00".encode("ascii") # encoding to ASCII for Python 3 _null2 = _null * 2 _null3 = _null * 3 -def guess_json_utf(data): +def guess_json_utf(data: bytes) -> typing.Optional[str]: """ :rtype: str """ @@ -863,68 +806,72 @@ def guess_json_utf(data): # determine the encoding. Also detect a BOM, if present. sample = data[:4] if sample in (codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE): - return 'utf-32' # BOM included + return "utf-32" # BOM included + if sample[:3] == codecs.BOM_UTF8: - return 'utf-8-sig' # BOM included, MS style (discouraged) + return "utf-8-sig" # BOM included, MS style (discouraged) + if sample[:2] in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE): - return 'utf-16' # BOM included + return "utf-16" # BOM included + nullcount = sample.count(_null) if nullcount == 0: - return 'utf-8' + return "utf-8" + if nullcount == 2: - if sample[::2] == _null2: # 1st and 3rd are null - return 'utf-16-be' + if sample[::2] == _null2: # 1st and 3rd are null + return "utf-16-be" + if sample[1::2] == _null2: # 2nd and 4th are null - return 'utf-16-le' - # Did not detect 2 valid UTF-16 ascii-range characters + return "utf-16-le" + + # Did not detect 2 valid UTF-16 ascii-range characters if nullcount == 3: if sample[:3] == _null3: - return 'utf-32-be' + return "utf-32-be" + if sample[1:] == _null3: - return 'utf-32-le' - # Did not detect a valid UTF-32 ascii-range character + return "utf-32-le" + + # Did not detect a valid UTF-32 ascii-range character return None -def prepend_scheme_if_needed(url, new_scheme): +def prepend_scheme_if_needed(url: str, new_scheme: str) -> str: """Given a URL that may or may not have a scheme, prepend the given scheme. Does not replace a present scheme with the one provided as an argument. :rtype: str """ scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme) - # urlparse is a finicky beast, and sometimes decides that there isn't a # netloc present. Assume that it's being over-cautious, and switch netloc # and path if urlparse decided there was no netloc. if not netloc: netloc, path = path, netloc - return urlunparse((scheme, netloc, path, params, query, fragment)) -def get_auth_from_url(url): +def get_auth_from_url(url: str) -> typing.Tuple[typing.Text, typing.Text]: """Given a url with authentication components, extract them into a tuple of username,password. :rtype: (str,str) """ parsed = urlparse(url) - try: auth = (unquote(parsed.username), unquote(parsed.password)) except (AttributeError, TypeError): - auth = ('', '') - + auth = ("", "") return auth # Moved outside of function to avoid recompile every call -_CLEAN_HEADER_REGEX_BYTE = re.compile(b'^\\S[^\\r\\n]*$|^$') -_CLEAN_HEADER_REGEX_STR = re.compile(r'^\S[^\r\n]*$|^$') +_CLEAN_HEADER_REGEX_BYTE = re.compile(b"^\\S[^\\r\\n]*$|^$") +_CLEAN_HEADER_REGEX_STR = re.compile(r"^\S[^\r\n]*$|^$") -def check_header_validity(header): +def check_header_validity(header: typing.Tuple[typing.Text, typing.Text]) -> None: """Verifies that header value is a string which doesn't contain leading whitespace or return characters. This prevents unintended header injection. @@ -932,46 +879,55 @@ def check_header_validity(header): :param header: tuple, in the format (name, value). """ name, value = header - - if isinstance(value, bytes): - pat = _CLEAN_HEADER_REGEX_BYTE - else: - pat = _CLEAN_HEADER_REGEX_STR + pat = _CLEAN_HEADER_REGEX_STR try: if not pat.match(value): - raise InvalidHeader("Invalid return character or leading space in header: %s" % name) + raise InvalidHeader( + "Invalid return character or leading space in header: %s" % name + ) + except TypeError: - raise InvalidHeader("Value for header {%s: %s} must be of type str or " - "bytes, not %s" % (name, value, type(value))) + raise InvalidHeader( + "Value for header {%s: %s} must be of type str or " + "bytes, not %s" % (name, value, type(value)) + ) -def urldefragauth(url): +def urldefragauth(url: str) -> str: """ Given a url remove the fragment and the authentication part. :rtype: str """ scheme, netloc, path, params, query, fragment = urlparse(url) - # see func:`prepend_scheme_if_needed` if not netloc: netloc, path = path, netloc - - netloc = netloc.rsplit('@', 1)[-1] - - return urlunparse((scheme, netloc, path, params, query, '')) + netloc = netloc.rsplit("@", 1)[-1] + return urlunparse((scheme, netloc, path, params, query, "")) -def rewind_body(prepared_request): +def rewind_body(prepared_request) -> None: """Move file pointer back to its recorded starting position so it can be read again on redirect. """ - body_seek = getattr(prepared_request.body, 'seek', None) - if body_seek is not None and isinstance(prepared_request._body_position, integer_types): + body_seek = getattr(prepared_request.body, "seek", None) + if body_seek is not None and isinstance( + prepared_request._body_position, integer_types + ): try: body_seek(prepared_request._body_position) except (IOError, OSError): - raise UnrewindableBodyError("An error occurred when rewinding request " - "body for redirect.") + raise UnrewindableBodyError( + "An error occurred when rewinding request " "body for redirect." + ) + else: raise UnrewindableBodyError("Unable to rewind request body for redirect.") + + +def is_stream(data: bytes) -> bool: + """Given data, determines if it should be sent as a stream.""" + is_iterable = getattr(data, "__iter__", False) + is_io_type = not isinstance(data, (basestring, list, tuple, collections.Mapping)) + return is_iterable and is_io_type diff --git a/setup.py b/setup.py index 10ce2c62..850f6e35 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # Learn more: https://github.com/kennethreitz/setup.py import os -import re import sys from codecs import open @@ -11,16 +10,27 @@ from setuptools.command.test import test as TestCommand here = os.path.abspath(os.path.dirname(__file__)) + +class Format(TestCommand): + user_options = [] + + def initialize_options(self): + TestCommand.initialize_options(self) + pass + + def finalize_options(self): + pass + + def run_tests(self): + os.system('white requests') + + class PyTest(TestCommand): user_options = [('pytest-args=', 'a', "Arguments to pass into py.test")] def initialize_options(self): TestCommand.initialize_options(self) - try: - from multiprocessing import cpu_count - self.pytest_args = ['-n', str(cpu_count()), '--boxed'] - except (ImportError, NotImplementedError): - self.pytest_args = ['-n', '1', '--boxed'] + self.pytest_args = ['-n', 'auto'] def finalize_options(self): TestCommand.finalize_options(self) @@ -33,16 +43,35 @@ class PyTest(TestCommand): errno = pytest.main(self.pytest_args) sys.exit(errno) + +class MyPyTest(TestCommand): + user_options = [('pytest-args=', 'a', "Arguments to pass into py.test")] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = ['-n', 'auto', '--mypy', 'tests'] + + def finalize_options(self): + TestCommand.finalize_options(self) + self.test_args = [] + self.test_suite = True + + def run_tests(self): + import pytest + + errno = pytest.main(self.pytest_args) + sys.exit(errno) + + # 'setup.py publish' shortcut. if sys.argv[-1] == 'publish': os.system('python setup.py sdist bdist_wheel') os.system('twine upload dist/*') sys.exit() - -packages = ['requests'] - +packages = ['requests3'] requires = [ 'chardet>=3.0.2,<3.1.0', +<<<<<<< HEAD 'idna>=2.5,<2.9', 'urllib3>=1.21.1,<1.25', 'certifi>=2017.4.17' @@ -62,10 +91,31 @@ with open(os.path.join(here, 'requests', '__version__.py'), 'r', 'utf-8') as f: exec(f.read(), about) with open('README.md', 'r', 'utf-8') as f: +======= + 'idna>=2.5,<2.7', + 'urllib3>=1.21.1,<1.23', + 'certifi>=2017.4.17', + 'rfc3986>=1.1.0<2', +] +test_requirements = [ + 'pytest-httpbin==0.0.7', + 'pytest-cov', + 'pytest-mock', + 'pytest-xdist', + 'PySocks>=1.5.6, !=1.5.7', + 'pytest>=2.8.0', + 'pytest-mypy', + 'mypy', + 'white', +] +about = {} +with open(os.path.join(here, 'requests3', '__version__.py'), 'r', 'utf-8') as f: + exec (f.read(), about) +with open('README.rst', 'r', 'utf-8') as f: +>>>>>>> 218d330150dbbe55f712296c2c39e0b4aa68b9a2 readme = f.read() with open('HISTORY.md', 'r', 'utf-8') as f: history = f.read() - setup( name=about['__title__'], version=about['__version__'], @@ -89,21 +139,36 @@ setup( 'Natural Language :: English', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', +<<<<<<< HEAD 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', +======= +>>>>>>> 218d330150dbbe55f712296c2c39e0b4aa68b9a2 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: CPython', +<<<<<<< HEAD 'Programming Language :: Python :: Implementation :: PyPy' ], cmdclass={'test': PyTest}, +======= + 'Programming Language :: Python :: Implementation :: PyPy', + ), + cmdclass={'test': PyTest, 'mypy': MyPyTest, 'format': Format}, +>>>>>>> 218d330150dbbe55f712296c2c39e0b4aa68b9a2 tests_require=test_requirements, extras_require={ 'security': ['pyOpenSSL >= 0.14', 'cryptography>=1.3.4', 'idna>=2.0.0'], 'socks': ['PySocks>=1.5.6, !=1.5.7'], +<<<<<<< HEAD 'socks:sys_platform == "win32" and python_version == "2.7"': ['win_inet_pton'], +======= + 'socks:sys_platform == "win32" and python_version == "2.7"': [ + 'win_inet_pton' + ], +>>>>>>> 218d330150dbbe55f712296c2c39e0b4aa68b9a2 }, ) diff --git a/tests/__init__.py b/tests/__init__.py index 9be94bcc..bb676233 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """Requests test package initialisation.""" import warnings diff --git a/tests/compat.py b/tests/compat.py index f68e8014..506331e2 100644 --- a/tests/compat.py +++ b/tests/compat.py @@ -1,21 +1,6 @@ # -*- coding: utf-8 -*- - -from requests.compat import is_py3 +import io as StringIO -try: - import StringIO -except ImportError: - import io as StringIO - -try: - from cStringIO import StringIO as cStringIO -except ImportError: - cStringIO = None - -if is_py3: - def u(s): - return s -else: - def u(s): - return s.decode('unicode-escape') +def u(s): + return s diff --git a/tests/conftest.py b/tests/conftest.py index cd64a765..79486a30 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- - import pytest -from requests.compat import urljoin +from urllib.parse import urljoin def prepare_url(value): diff --git a/tests/test_help.py b/tests/test_help.py index 3beb65f3..d45fba72 100644 --- a/tests/test_help.py +++ b/tests/test_help.py @@ -1,15 +1,23 @@ # -*- encoding: utf-8 - import sys import pytest -from requests.help import info +from requests3.help import info +@pytest.mark.skipif(sys.version_info[:2] != (2, 6), reason="Only run on Python 2.6") +def test_system_ssl_py26(): + """OPENSSL_VERSION_NUMBER isn't provided in Python 2.6, verify we don't + blow up in this case. + """ + assert info()["system_ssl"] == {"version": ""} + + +@pytest.mark.skipif(sys.version_info < (2, 7), reason="Only run on Python 2.7+") def test_system_ssl(): """Verify we're actually setting system_ssl when it should be available.""" - assert info()['system_ssl']['version'] != '' + assert info()["system_ssl"]["version"] != "" class VersionedPackage(object): @@ -21,11 +29,11 @@ def test_idna_without_version_attribute(mocker): """Older versions of IDNA don't provide a __version__ attribute, verify that if we have such a package, we don't blow up. """ - mocker.patch('requests.help.idna', new=None) - assert info()['idna'] == {'version': ''} + mocker.patch("requests.help.idna", new=None) + assert info()["idna"] == {"version": ""} def test_idna_with_version_attribute(mocker): """Verify we're actually setting idna version when it should be available.""" - mocker.patch('requests.help.idna', new=VersionedPackage('2.6')) - assert info()['idna'] == {'version': '2.6'} + mocker.patch("requests.help.idna", new=VersionedPackage("2.6")) + assert info()["idna"] == {"version": "2.6"} diff --git a/tests/test_hooks.py b/tests/test_hooks.py index 014b4391..b8e7f36e 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- - import pytest -from requests import hooks +from requests3 import hooks def hook(value): @@ -10,13 +9,12 @@ def hook(value): @pytest.mark.parametrize( - 'hooks_list, result', ( - (hook, 'ata'), - ([hook, lambda x: None, hook], 'ta'), - ) + 'hooks_list, result', ((hook, 'ata'), ([hook, lambda x: None, hook], 'ta')) ) def test_hooks(hooks_list, result): - assert hooks.dispatch_hook('response', {'response': hooks_list}, 'Data') == result + assert hooks.dispatch_hook( + 'response', {'response': hooks_list}, 'Data' + ) == result def test_default_hooks(): diff --git a/tests/test_lowlevel.py b/tests/test_lowlevel.py index 82c3b25a..d4c221ca 100644 --- a/tests/test_lowlevel.py +++ b/tests/test_lowlevel.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- - import pytest import threading -import requests +import requests3 as requests from tests.testserver.server import Server, consume_socket_content @@ -13,15 +12,29 @@ def test_chunked_upload(): """can safely send generators""" close_server = threading.Event() server = Server.basic_response_server(wait_to_close_event=close_server) - data = iter([b'a', b'b', b'c']) - + data = iter([b"a", b"b", b"c"]) with server as (host, port): - url = 'http://{}:{}/'.format(host, port) + url = "http://{}:{}/".format(host, port) r = requests.post(url, data=data, stream=True) close_server.set() # release server block - assert r.status_code == 200 - assert r.request.headers['Transfer-Encoding'] == 'chunked' + assert r.request.headers["Transfer-Encoding"] == "chunked" + + +def test_incorrect_content_length(): + """Test ConnectionError raised for incomplete responses""" + close_server = threading.Event() + server = Server.text_response_server( + "HTTP/1.1 200 OK\r\n" + "Content-Length: 50\r\n\r\n" + "Hello World." + ) + with server as (host, port): + url = "http://{0}:{1}/".format(host, port) + r = requests.Request("GET", url).prepare() + s = requests.Session() + with pytest.raises(requests.exceptions.ConnectionError) as e: + resp = s.send(r) + assert "12 bytes read, 38 more expected" in str(e) + close_server.set() # release server block def test_digestauth_401_count_reset_on_redirect(): @@ -30,60 +43,52 @@ def test_digestauth_401_count_reset_on_redirect(): See https://github.com/requests/requests/issues/1979. """ - text_401 = (b'HTTP/1.1 401 UNAUTHORIZED\r\n' - b'Content-Length: 0\r\n' - b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"' - b', opaque="372825293d1c26955496c80ed6426e9e", ' - b'realm="me@kennethreitz.com", qop=auth\r\n\r\n') - - text_302 = (b'HTTP/1.1 302 FOUND\r\n' - b'Content-Length: 0\r\n' - b'Location: /\r\n\r\n') - - text_200 = (b'HTTP/1.1 200 OK\r\n' - b'Content-Length: 0\r\n\r\n') - - expected_digest = (b'Authorization: Digest username="user", ' - b'realm="me@kennethreitz.com", ' - b'nonce="6bf5d6e4da1ce66918800195d6b9130d", uri="/"') - - auth = requests.auth.HTTPDigestAuth('user', 'pass') + text_401 = ( + b"HTTP/1.1 401 UNAUTHORIZED\r\n" + b"Content-Length: 0\r\n" + b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"' + b', opaque="372825293d1c26955496c80ed6426e9e", ' + b'realm="me@kennethreitz.com", qop=auth\r\n\r\n' + ) + text_302 = b"HTTP/1.1 302 FOUND\r\n" b"Content-Length: 0\r\n" b"Location: /\r\n\r\n" + text_200 = b"HTTP/1.1 200 OK\r\n" b"Content-Length: 0\r\n\r\n" + expected_digest = ( + b'Authorization: Digest username="user", ' + b'realm="me@kennethreitz.com", ' + b'nonce="6bf5d6e4da1ce66918800195d6b9130d", uri="/"' + ) + auth = requests.auth.HTTPDigestAuth("user", "pass") def digest_response_handler(sock): # Respond to initial GET with a challenge. request_content = consume_socket_content(sock, timeout=0.5) assert request_content.startswith(b"GET / HTTP/1.1") sock.send(text_401) - # Verify we receive an Authorization header in response, then redirect. request_content = consume_socket_content(sock, timeout=0.5) assert expected_digest in request_content sock.send(text_302) - # Verify Authorization isn't sent to the redirected host, # then send another challenge. request_content = consume_socket_content(sock, timeout=0.5) - assert b'Authorization:' not in request_content + assert b"Authorization:" not in request_content sock.send(text_401) - # Verify Authorization is sent correctly again, and return 200 OK. request_content = consume_socket_content(sock, timeout=0.5) assert expected_digest in request_content sock.send(text_200) - return request_content close_server = threading.Event() server = Server(digest_response_handler, wait_to_close_event=close_server) - with server as (host, port): - url = 'http://{}:{}/'.format(host, port) + url = "http://{}:{}/".format(host, port) r = requests.get(url, auth=auth) # Verify server succeeded in authenticating. assert r.status_code == 200 # Verify Authorization was sent in final request. - assert 'Authorization' in r.request.headers - assert r.request.headers['Authorization'].startswith('Digest ') + assert "Authorization" in r.request.headers + assert r.request.headers["Authorization"].startswith("Digest ") # Verify redirect happened as we expected. assert r.history[0].status_code == 302 close_server.set() @@ -93,41 +98,39 @@ def test_digestauth_401_only_sent_once(): """Ensure we correctly respond to a 401 challenge once, and then stop responding if challenged again. """ - text_401 = (b'HTTP/1.1 401 UNAUTHORIZED\r\n' - b'Content-Length: 0\r\n' - b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"' - b', opaque="372825293d1c26955496c80ed6426e9e", ' - b'realm="me@kennethreitz.com", qop=auth\r\n\r\n') - - expected_digest = (b'Authorization: Digest username="user", ' - b'realm="me@kennethreitz.com", ' - b'nonce="6bf5d6e4da1ce66918800195d6b9130d", uri="/"') - - auth = requests.auth.HTTPDigestAuth('user', 'pass') + text_401 = ( + b"HTTP/1.1 401 UNAUTHORIZED\r\n" + b"Content-Length: 0\r\n" + b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"' + b', opaque="372825293d1c26955496c80ed6426e9e", ' + b'realm="me@kennethreitz.com", qop=auth\r\n\r\n' + ) + expected_digest = ( + b'Authorization: Digest username="user", ' + b'realm="me@kennethreitz.com", ' + b'nonce="6bf5d6e4da1ce66918800195d6b9130d", uri="/"' + ) + auth = requests.auth.HTTPDigestAuth("user", "pass") def digest_failed_response_handler(sock): # Respond to initial GET with a challenge. request_content = consume_socket_content(sock, timeout=0.5) assert request_content.startswith(b"GET / HTTP/1.1") sock.send(text_401) - # Verify we receive an Authorization header in response, then # challenge again. request_content = consume_socket_content(sock, timeout=0.5) assert expected_digest in request_content sock.send(text_401) - # Verify the client didn't respond to second challenge. request_content = consume_socket_content(sock, timeout=0.5) - assert request_content == b'' - + assert request_content == b"" return request_content close_server = threading.Event() server = Server(digest_failed_response_handler, wait_to_close_event=close_server) - with server as (host, port): - url = 'http://{}:{}/'.format(host, port) + url = "http://{}:{}/".format(host, port) r = requests.get(url, auth=auth) # Verify server didn't authenticate us. assert r.status_code == 401 @@ -140,31 +143,29 @@ def test_digestauth_only_on_4xx(): See https://github.com/requests/requests/issues/3772. """ - text_200_chal = (b'HTTP/1.1 200 OK\r\n' - b'Content-Length: 0\r\n' - b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"' - b', opaque="372825293d1c26955496c80ed6426e9e", ' - b'realm="me@kennethreitz.com", qop=auth\r\n\r\n') - - auth = requests.auth.HTTPDigestAuth('user', 'pass') + text_200_chal = ( + b"HTTP/1.1 200 OK\r\n" + b"Content-Length: 0\r\n" + b'WWW-Authenticate: Digest nonce="6bf5d6e4da1ce66918800195d6b9130d"' + b', opaque="372825293d1c26955496c80ed6426e9e", ' + b'realm="me@kennethreitz.com", qop=auth\r\n\r\n' + ) + auth = requests.auth.HTTPDigestAuth("user", "pass") def digest_response_handler(sock): # Respond to GET with a 200 containing www-authenticate header. request_content = consume_socket_content(sock, timeout=0.5) assert request_content.startswith(b"GET / HTTP/1.1") sock.send(text_200_chal) - # Verify the client didn't respond with auth. request_content = consume_socket_content(sock, timeout=0.5) - assert request_content == b'' - + assert request_content == b"" return request_content close_server = threading.Event() server = Server(digest_response_handler, wait_to_close_event=close_server) - with server as (host, port): - url = 'http://{}:{}/'.format(host, port) + url = "http://{}:{}/".format(host, port) r = requests.get(url, auth=auth) # Verify server didn't receive auth from us. assert r.status_code == 200 @@ -173,22 +174,20 @@ def test_digestauth_only_on_4xx(): _schemes_by_var_prefix = [ - ('http', ['http']), - ('https', ['https']), - ('all', ['http', 'https']), + ("http", ["http"]), + ("https", ["https"]), + ("all", ["http", "https"]), ] - _proxy_combos = [] for prefix, schemes in _schemes_by_var_prefix: for scheme in schemes: - _proxy_combos.append(("{}_proxy".format(prefix), scheme)) - + _proxy_combos.append(("{0}_proxy".format(prefix), scheme)) _proxy_combos += [(var.upper(), scheme) for var, scheme in _proxy_combos] @pytest.mark.parametrize("var,scheme", _proxy_combos) def test_use_proxy_from_environment(httpbin, var, scheme): - url = "{}://httpbin.org".format(scheme) + url = "{0}://httpbin.org".format(scheme) fake_proxy = Server() # do nothing with the requests; just close the socket with fake_proxy as (host, port): proxy_url = "socks5://{}:{}".format(host, port) @@ -197,113 +196,37 @@ def test_use_proxy_from_environment(httpbin, var, scheme): # fake proxy's lack of response will cause a ConnectionError with pytest.raises(requests.exceptions.ConnectionError): requests.get(url) - # the fake proxy received a request assert len(fake_proxy.handler_results) == 1 - # it had actual content (not checking for SOCKS protocol for now) assert len(fake_proxy.handler_results[0]) > 0 def test_redirect_rfc1808_to_non_ascii_location(): - path = u'š' - expected_path = b'%C5%A1' + path = u"š" + expected_path = b"%C5%A1" redirect_request = [] # stores the second request to the server def redirect_resp_handler(sock): consume_socket_content(sock, timeout=0.5) - location = u'//{}:{}/{}'.format(host, port, path) + location = u"//{}:{}/{}".format(host, port, path) sock.send( - b'HTTP/1.1 301 Moved Permanently\r\n' - b'Content-Length: 0\r\n' - b'Location: ' + location.encode('utf8') + b'\r\n' - b'\r\n' + b"HTTP/1.1 301 Moved Permanently\r\n" + b"Content-Length: 0\r\n" + b"Location: " + location.encode("utf8") + b"\r\n" + b"\r\n" ) redirect_request.append(consume_socket_content(sock, timeout=0.5)) - sock.send(b'HTTP/1.1 200 OK\r\n\r\n') + sock.send(b"HTTP/1.1 200 OK\r\n\r\n") close_server = threading.Event() server = Server(redirect_resp_handler, wait_to_close_event=close_server) - with server as (host, port): - url = u'http://{}:{}'.format(host, port) + url = u"http://{}:{}".format(host, port) r = requests.get(url=url, allow_redirects=True) assert r.status_code == 200 assert len(r.history) == 1 assert r.history[0].status_code == 301 - assert redirect_request[0].startswith(b'GET /' + expected_path + b' HTTP/1.1') - assert r.url == u'{}/{}'.format(url, expected_path.decode('ascii')) - - close_server.set() - -def test_fragment_not_sent_with_request(): - """Verify that the fragment portion of a URI isn't sent to the server.""" - def response_handler(sock): - req = consume_socket_content(sock, timeout=0.5) - sock.send( - b'HTTP/1.1 200 OK\r\n' - b'Content-Length: '+bytes(len(req))+b'\r\n' - b'\r\n'+req - ) - - close_server = threading.Event() - server = Server(response_handler, wait_to_close_event=close_server) - - with server as (host, port): - url = 'http://{}:{}/path/to/thing/#view=edit&token=hunter2'.format(host, port) - r = requests.get(url) - raw_request = r.content - - assert r.status_code == 200 - headers, body = raw_request.split(b'\r\n\r\n', 1) - status_line, headers = headers.split(b'\r\n', 1) - - assert status_line == b'GET /path/to/thing/ HTTP/1.1' - for frag in (b'view', b'edit', b'token', b'hunter2'): - assert frag not in headers - assert frag not in body - - close_server.set() - -def test_fragment_update_on_redirect(): - """Verify we only append previous fragment if one doesn't exist on new - location. If a new fragment is encountered in a Location header, it should - be added to all subsequent requests. - """ - - def response_handler(sock): - consume_socket_content(sock, timeout=0.5) - sock.send( - b'HTTP/1.1 302 FOUND\r\n' - b'Content-Length: 0\r\n' - b'Location: /get#relevant-section\r\n\r\n' - ) - consume_socket_content(sock, timeout=0.5) - sock.send( - b'HTTP/1.1 302 FOUND\r\n' - b'Content-Length: 0\r\n' - b'Location: /final-url/\r\n\r\n' - ) - consume_socket_content(sock, timeout=0.5) - sock.send( - b'HTTP/1.1 200 OK\r\n\r\n' - ) - - close_server = threading.Event() - server = Server(response_handler, wait_to_close_event=close_server) - - with server as (host, port): - url = 'http://{}:{}/path/to/thing/#view=edit&token=hunter2'.format(host, port) - r = requests.get(url) - raw_request = r.content - - assert r.status_code == 200 - assert len(r.history) == 2 - assert r.history[0].request.url == url - - # Verify we haven't overwritten the location with our previous fragment. - assert r.history[1].request.url == 'http://{}:{}/get#relevant-section'.format(host, port) - # Verify previous fragment is used and not the original. - assert r.url == 'http://{}:{}/final-url/#relevant-section'.format(host, port) - + assert redirect_request[0].startswith(b"GET /" + expected_path + b" HTTP/1.1") + assert r.url == u"{0}/{1}".format(url, expected_path.decode("ascii")) close_server.set() diff --git a/tests/test_packages.py b/tests/test_packages.py deleted file mode 100644 index b55cb68c..00000000 --- a/tests/test_packages.py +++ /dev/null @@ -1,13 +0,0 @@ -import requests - - -def test_can_access_urllib3_attribute(): - requests.packages.urllib3 - - -def test_can_access_idna_attribute(): - requests.packages.idna - - -def test_can_access_chardet_attribute(): - requests.packages.chardet diff --git a/tests/test_requests.py b/tests/test_requests.py index 89eff885..1d7b862c 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- - """Tests for Requests.""" from __future__ import division +import itertools import json import os import pickle @@ -14,80 +14,108 @@ import re import io import requests import pytest -from requests.adapters import HTTPAdapter -from requests.auth import HTTPDigestAuth, _basic_auth_str -from requests.compat import ( - Morsel, cookielib, getproxies, str, urlparse, - builtin_str, OrderedDict) -from requests.cookies import ( - cookiejar_from_dict, morsel_to_cookie) -from requests.exceptions import ( - ConnectionError, ConnectTimeout, InvalidSchema, InvalidURL, - MissingSchema, ReadTimeout, Timeout, RetryError, TooManyRedirects, - ProxyError, InvalidHeader, UnrewindableBodyError, SSLError, InvalidProxyURL) -from requests.models import PreparedRequest -from requests.structures import CaseInsensitiveDict -from requests.sessions import SessionRedirectMixin -from requests.models import urlencode -from requests.hooks import default_hooks -from requests.compat import MutableMapping +import pytest_httpbin +from requests3.adapters import HTTPAdapter +from requests3.auth import HTTPDigestAuth, _basic_auth_str +from requests3.basics import ( + Morsel, cookielib, getproxies, str, urlparse, builtin_str +) +from requests3.cookies import ( cookiejar_from_dict, morsel_to_cookie) +from requests3.exceptions import ( + ConnectionError, + ConnectTimeout, + InvalidScheme, + InvalidURL, + MissingScheme, + ReadTimeout, + Timeout, + RetryError, + TooManyRedirects, + ProxyError, + InvalidHeader, + UnrewindableBodyError, + InvalidBodyError, + SSLError, +) +from requests3.models import PreparedRequest +from requests3.structures import CaseInsensitiveDict +from requests3.sessions import SessionRedirectMixin +from requests3.models import urlencode +from requests3.hooks import default_hooks +from requests3.utils import DEFAULT_CA_BUNDLE_PATH from .compat import StringIO, u from .utils import override_environ from urllib3.util import Timeout as Urllib3Timeout + +class SendRecordingAdapter(HTTPAdapter): + """ + A basic subclass of the HTTPAdapter that records the arguments used to + ``send``. + """ + + def __init__(self, *args, **kwargs): + super(SendRecordingAdapter, self).__init__(*args, **kwargs) + self.send_calls = [] + + def send(self, *args, **kwargs): + self.send_calls.append((args, kwargs)) + return super(SendRecordingAdapter, self).send(*args, **kwargs) + + # Requests to this URL should always fail with a connection timeout (nothing # listening on that port) TARPIT = 'http://10.255.255.1' - try: from ssl import SSLContext + del SSLContext HAS_MODERN_SSL = True except ImportError: HAS_MODERN_SSL = False - try: requests.pyopenssl HAS_PYOPENSSL = True except AttributeError: HAS_PYOPENSSL = False +@pytest.fixture +def s(request, *args, **kwargs): + return requests.Session() + class TestRequests: digest_auth_algo = ('MD5', 'SHA-256', 'SHA-512') def test_entry_points(self): - - requests.session - requests.session().get - requests.session().head + requests.Session().get + requests.Session().head requests.get requests.head requests.put requests.patch requests.post - # Not really an entry point, but people rely on it. - from requests.packages.urllib3.poolmanager import PoolManager @pytest.mark.parametrize( - 'exception, url', ( - (MissingSchema, 'hiwpefhipowhefopw'), - (InvalidSchema, 'localhost:3128'), - (InvalidSchema, 'localhost.localdomain:3128/'), - (InvalidSchema, '10.122.1.1:3128/'), + 'exception, url', + ( + (MissingScheme, 'hiwpefhipowhefopw'), + (InvalidScheme, 'localhost:3128'), + (InvalidScheme, 'localhost.localdomain:3128/'), + (InvalidScheme, '10.122.1.1:3128/'), (InvalidURL, 'http://'), - )) + ), + ) def test_invalid_url(self, exception, url): with pytest.raises(exception): requests.get(url) def test_basic_building(self): - req = requests.Request() + req = requests.Request(method='GET') req.url = 'http://kennethreitz.org/' req.data = {'life': '42'} - pr = req.prepare() assert pr.url == req.url assert pr.body == 'life=42' @@ -104,46 +132,65 @@ class TestRequests: @pytest.mark.parametrize('method', ('POST', 'PUT', 'PATCH', 'OPTIONS')) def test_empty_content_length(self, httpbin, method): - req = requests.Request(method, httpbin(method.lower()), data='').prepare() + req = requests.Request( + method, httpbin(method.lower()), data='' + ).prepare( + ) assert req.headers['Content-Length'] == '0' def test_override_content_length(self, httpbin): - headers = { - 'Content-Length': 'not zero' - } - r = requests.Request('POST', httpbin('post'), headers=headers).prepare() + headers = {'Content-Length': 'not zero'} + r = requests.Request('POST', httpbin('post'), headers=headers).prepare( + ) assert 'Content-Length' in r.headers assert r.headers['Content-Length'] == 'not zero' def test_path_is_not_double_encoded(self): - request = requests.Request('GET', "http://0.0.0.0/get/test case").prepare() - + request = requests.Request( + 'GET', "http://0.0.0.0/get/test case" + ).prepare( + ) assert request.path_url == '/get/test%20case' @pytest.mark.parametrize( - 'url, expected', ( - ('http://example.com/path#fragment', 'http://example.com/path?a=b#fragment'), - ('http://example.com/path?key=value#fragment', 'http://example.com/path?key=value&a=b#fragment') - )) + 'url, expected', + ( + ( + 'http://example.com/path#fragment', + 'http://example.com/path?a=b#fragment', + ), + ( + 'http://example.com/path?key=value#fragment', + 'http://example.com/path?key=value&a=b#fragment', + ), + ), + ) def test_params_are_added_before_fragment(self, url, expected): request = requests.Request('GET', url, params={"a": "b"}).prepare() assert request.url == expected - def test_params_original_order_is_preserved_by_default(self): - param_ordered_dict = OrderedDict((('z', 1), ('a', 1), ('k', 1), ('d', 1))) - session = requests.Session() - request = requests.Request('GET', 'http://example.com/', params=param_ordered_dict) - prep = session.prepare_request(request) + def test_params_original_order_is_preserved_by_default(self, s): + param_ordered_dict = collections.OrderedDict( + (('z', 1), ('a', 1), ('k', 1), ('d', 1)) + ) + request = requests.Request( + 'GET', 'http://example.com/', params=param_ordered_dict + ) + prep = s.prepare_request(request) assert prep.url == 'http://example.com/?z=1&a=1&k=1&d=1' - def test_params_bytes_are_encoded(self): - request = requests.Request('GET', 'http://example.com', - params=b'test=foo').prepare() - assert request.url == 'http://example.com/?test=foo' + # def test_params_bytes_are_encoded(self): + # request = requests.Request( + # 'GET', 'http://example.com', params=b'test=foo' + # ).prepare( + # ) + # assert request.url == 'http://example.com/?test=foo' def test_binary_put(self): - request = requests.Request('PUT', 'http://example.com', - data=u"ööö".encode("utf-8")).prepare() + request = requests.Request( + 'PUT', 'http://example.com', data=u"ööö".encode("utf-8") + ).prepare( + ) assert isinstance(request.body, bytes) def test_whitespaces_are_removed_from_url(self): @@ -151,9 +198,10 @@ class TestRequests: request = requests.Request('GET', ' http://example.com').prepare() assert request.url == 'http://example.com/' - @pytest.mark.parametrize('scheme', ('http://', 'HTTP://', 'hTTp://', 'HttP://')) - def test_mixed_case_scheme_acceptable(self, httpbin, scheme): - s = requests.Session() + @pytest.mark.parametrize( + 'scheme', ('http://', 'HTTP://', 'hTTp://', 'HttP://') + ) + def test_mixed_case_scheme_acceptable(self, s, httpbin, scheme): s.proxies = getproxies() parts = urlparse(httpbin('get')) url = scheme + parts.netloc + parts.path @@ -161,13 +209,10 @@ class TestRequests: r = s.send(r.prepare()) assert r.status_code == 200, 'failed for scheme {}'.format(scheme) - def test_HTTP_200_OK_GET_ALTERNATIVE(self, httpbin): + def test_HTTP_200_OK_GET_ALTERNATIVE(self, s, httpbin): r = requests.Request('GET', httpbin('get')) - s = requests.Session() s.proxies = getproxies() - r = s.send(r.prepare()) - assert r.status_code == 200 def test_HTTP_302_ALLOW_REDIRECT_GET(self, httpbin): @@ -177,7 +222,11 @@ class TestRequests: assert r.history[0].is_redirect def test_HTTP_307_ALLOW_REDIRECT_POST(self, httpbin): - r = requests.post(httpbin('redirect-to'), data='test', params={'url': 'post', 'status_code': 307}) + r = requests.post( + httpbin('redirect-to'), + data='test', + params={'url': 'post', 'status_code': 307}, + ) assert r.status_code == 200 assert r.history[0].status_code == 307 assert r.history[0].is_redirect @@ -185,7 +234,11 @@ class TestRequests: def test_HTTP_307_ALLOW_REDIRECT_POST_WITH_SEEKABLE(self, httpbin): byte_str = b'test' - r = requests.post(httpbin('redirect-to'), data=io.BytesIO(byte_str), params={'url': 'post', 'status_code': 307}) + r = requests.post( + httpbin('redirect-to'), + data=io.BytesIO(byte_str), + params={'url': 'post', 'status_code': 307}, + ) assert r.status_code == 200 assert r.history[0].status_code == 307 assert r.history[0].is_redirect @@ -200,10 +253,11 @@ class TestRequests: assert e.response.url == url assert len(e.response.history) == 30 else: - pytest.fail('Expected redirect to raise TooManyRedirects but it did not') + pytest.fail( + 'Expected redirect to raise TooManyRedirects but it did not' + ) - def test_HTTP_302_TOO_MANY_REDIRECTS_WITH_PARAMS(self, httpbin): - s = requests.session() + def test_HTTP_302_TOO_MANY_REDIRECTS_WITH_PARAMS(self, s, httpbin): s.max_redirects = 5 try: s.get(httpbin('relative-redirect', '50')) @@ -213,87 +267,155 @@ class TestRequests: assert e.response.url == url assert len(e.response.history) == 5 else: - pytest.fail('Expected custom max number of redirects to be respected but was not') + pytest.fail( + 'Expected custom max number of redirects to be respected but was not' + ) - def test_http_301_changes_post_to_get(self, httpbin): - r = requests.post(httpbin('status', '301')) - assert r.status_code == 200 - assert r.request.method == 'GET' + @pytest.mark.parametrize( + 'method, body, expected', + ( + ('GET', None, 'GET'), + ('HEAD', None, 'HEAD'), + ('POST', 'test', 'GET'), + ('PUT', 'put test', 'PUT'), + ('PATCH', 'patch test', 'PATCH'), + ('DELETE', '', 'DELETE'), + ), + ) + def test_http_301_for_redirectable_methods( + self, httpbin, method, body, expected + ): + """Tests all methods except OPTIONS for expected redirect behaviour. + + OPTIONS responses can behave differently depending on the server, so + we don't have anything uniform to test except how httpbin responds + to them. For that reason they aren't included here. + """ + params = {'url': '/%s' % expected.lower(), 'status_code': '301'} + r = requests.request( + method, httpbin('redirect-to'), data=body, params=params + ) + assert r.request.url == httpbin(expected.lower()) + assert r.request.method == expected assert r.history[0].status_code == 301 assert r.history[0].is_redirect + if expected in ('GET', 'HEAD'): + assert r.request.body is None + else: + assert r.json()['data'] == body - def test_http_301_doesnt_change_head_to_get(self, httpbin): - r = requests.head(httpbin('status', '301'), allow_redirects=True) - print(r.content) - assert r.status_code == 200 - assert r.request.method == 'HEAD' - assert r.history[0].status_code == 301 - assert r.history[0].is_redirect + @pytest.mark.parametrize( + 'method, body, expected', + ( + ('GET', None, 'GET'), + ('HEAD', None, 'HEAD'), + ('POST', 'test', 'GET'), + ('PUT', 'put test', 'PUT'), + ('PATCH', 'patch test', 'PATCH'), + ('DELETE', '', 'DELETE'), + ), + ) + def test_http_302_for_redirectable_methods( + self, httpbin, method, body, expected + ): + """Tests all methods except OPTIONS for expected redirect behaviour. - def test_http_302_changes_post_to_get(self, httpbin): - r = requests.post(httpbin('status', '302')) - assert r.status_code == 200 - assert r.request.method == 'GET' + OPTIONS responses can behave differently depending on the server, so + we don't have anything uniform to test except how httpbin responds + to them. For that reason they aren't included here. + """ + params = {'url': '/%s' % expected.lower()} + r = requests.request( + method, httpbin('redirect-to'), data=body, params=params + ) + assert r.request.url == httpbin(expected.lower()) + assert r.request.method == expected assert r.history[0].status_code == 302 assert r.history[0].is_redirect + if expected in ('GET', 'HEAD'): + assert r.request.body is None + else: + assert r.json()['data'] == body - def test_http_302_doesnt_change_head_to_get(self, httpbin): - r = requests.head(httpbin('status', '302'), allow_redirects=True) - assert r.status_code == 200 - assert r.request.method == 'HEAD' - assert r.history[0].status_code == 302 - assert r.history[0].is_redirect + @pytest.mark.parametrize( + 'method, body, expected', + ( + ('GET', None, 'GET'), + ('HEAD', None, 'HEAD'), + ('POST', 'test', 'GET'), + ('PUT', 'put test', 'GET'), + ('PATCH', 'patch test', 'GET'), + ('DELETE', '', 'GET'), + ), + ) + def test_http_303_for_redirectable_methods( + self, httpbin, method, body, expected + ): + """Tests all methods except OPTIONS for expected redirect behaviour. - def test_http_303_changes_post_to_get(self, httpbin): - r = requests.post(httpbin('status', '303')) - assert r.status_code == 200 - assert r.request.method == 'GET' + OPTIONS responses can behave differently depending on the server, so + we don't have anything uniform to test except how httpbin responds + to them. For that reason they aren't included here. + """ + params = {'url': '/%s' % expected.lower(), 'status_code': '303'} + r = requests.request( + method, httpbin('redirect-to'), data=body, params=params + ) + assert r.request.url == httpbin(expected.lower()) + assert r.request.method == expected assert r.history[0].status_code == 303 assert r.history[0].is_redirect + assert r.request.body is None - def test_http_303_doesnt_change_head_to_get(self, httpbin): - r = requests.head(httpbin('status', '303'), allow_redirects=True) - assert r.status_code == 200 - assert r.request.method == 'HEAD' - assert r.history[0].status_code == 303 - assert r.history[0].is_redirect + def test_multiple_location_headers(self, s, httpbin): + headers = [ + ('Location', 'http://example.com'), + ('Location', 'https://example.com/1'), + ] + params = '&'.join(['%s=%s' % (k, v) for k, v in headers]) - def test_header_and_body_removal_on_redirect(self, httpbin): + req = requests.Request('GET', httpbin('response-headers?%s' % params)) + prep = s.prepare_request(req) + resp = s.send(prep) + # change response to redirect + resp.status_code = 302 + with pytest.raises(InvalidHeader): + # next triggers yield on generator + next(s.resolve_redirects(resp, prep)) + + def test_header_and_body_removal_on_redirect(self, s, httpbin): purged_headers = ('Content-Length', 'Content-Type') - ses = requests.Session() - req = requests.Request('POST', httpbin('post'), data={'test': 'data'}) - prep = ses.prepare_request(req) - resp = ses.send(prep) + req = requests.Request('POST', httpbin('post'), data={'test': 'data'}) + prep = s.prepare_request(req) + resp = s.send(prep) # Mimic a redirect response resp.status_code = 302 resp.headers['location'] = 'get' - # Run request through resolve_redirects - next_resp = next(ses.resolve_redirects(resp, prep)) + next_resp = next(s.resolve_redirects(resp, prep)) assert next_resp.request.body is None for header in purged_headers: assert header not in next_resp.request.headers - def test_transfer_enc_removal_on_redirect(self, httpbin): + def test_transfer_enc_removal_on_redirect(self, s, httpbin): purged_headers = ('Transfer-Encoding', 'Content-Type') - ses = requests.Session() - req = requests.Request('POST', httpbin('post'), data=(b'x' for x in range(1))) - prep = ses.prepare_request(req) - assert 'Transfer-Encoding' in prep.headers + req = requests.Request( + 'POST', httpbin('post'), data=(b'x' for x in range(1)) + ) + prep = s.prepare_request(req) + assert 'Transfer-Encoding' in prep.headers # Create Response to avoid https://github.com/kevin1024/pytest-httpbin/issues/33 resp = requests.Response() resp.raw = io.BytesIO(b'the content') resp.request = prep setattr(resp.raw, 'release_conn', lambda *args: args) - # Mimic a redirect response resp.status_code = 302 resp.headers['location'] = httpbin('get') - # Run request through resolve_redirect - next_resp = next(ses.resolve_redirects(resp, prep)) + next_resp = next(s.resolve_redirects(resp, prep)) assert next_resp.request.body is None for header in purged_headers: assert header not in next_resp.request.headers @@ -308,72 +430,62 @@ class TestRequests: def test_HTTP_200_OK_GET_WITH_PARAMS(self, httpbin): heads = {'User-agent': 'Mozilla/5.0'} - r = requests.get(httpbin('user-agent'), headers=heads) - assert heads['User-agent'] in r.text assert r.status_code == 200 def test_HTTP_200_OK_GET_WITH_MIXED_PARAMS(self, httpbin): heads = {'User-agent': 'Mozilla/5.0'} - - r = requests.get(httpbin('get') + '?test=true', params={'q': 'test'}, headers=heads) + r = requests.get( + httpbin('get') + '?test=true', params={'q': 'test'}, headers=heads + ) assert r.status_code == 200 - def test_set_cookie_on_301(self, httpbin): - s = requests.session() + def test_set_cookie_on_301(self, s, httpbin): url = httpbin('cookies/set?foo=bar') s.get(url) assert s.cookies['foo'] == 'bar' - def test_cookie_sent_on_redirect(self, httpbin): - s = requests.session() + def test_cookie_sent_on_redirect(self, s, httpbin): s.get(httpbin('cookies/set?foo=bar')) r = s.get(httpbin('redirect/1')) # redirects to httpbin('get') assert 'Cookie' in r.json()['headers'] - def test_cookie_removed_on_expire(self, httpbin): - s = requests.session() + def test_cookie_removed_on_expire(self, s, httpbin): s.get(httpbin('cookies/set?foo=bar')) assert s.cookies['foo'] == 'bar' s.get( httpbin('response-headers'), params={ - 'Set-Cookie': - 'foo=deleted; expires=Thu, 01-Jan-1970 00:00:01 GMT' - } + 'Set-Cookie': 'foo=deleted; expires=Thu, 01-Jan-1970 00:00:01 GMT' + }, ) assert 'foo' not in s.cookies - def test_cookie_quote_wrapped(self, httpbin): - s = requests.session() + def test_cookie_quote_wrapped(self, s, httpbin): s.get(httpbin('cookies/set?foo="bar:baz"')) assert s.cookies['foo'] == '"bar:baz"' - def test_cookie_persists_via_api(self, httpbin): - s = requests.session() + def test_cookie_persists_via_api(self, s, httpbin): r = s.get(httpbin('redirect/1'), cookies={'foo': 'bar'}) assert 'foo' in r.request.headers['Cookie'] assert 'foo' in r.history[0].request.headers['Cookie'] - def test_request_cookie_overrides_session_cookie(self, httpbin): - s = requests.session() + def test_request_cookie_overrides_session_cookie(self, s, httpbin): s.cookies['foo'] = 'bar' r = s.get(httpbin('cookies'), cookies={'foo': 'baz'}) assert r.json()['cookies']['foo'] == 'baz' # Session cookie should not be modified assert s.cookies['foo'] == 'bar' - def test_request_cookies_not_persisted(self, httpbin): - s = requests.session() + def test_request_cookies_not_persisted(self, s, httpbin): s.get(httpbin('cookies'), cookies={'foo': 'baz'}) # Sending a request with cookies should not add cookies to the session assert not s.cookies - def test_generic_cookiejar_works(self, httpbin): + def test_generic_cookiejar_works(self, s, httpbin): cj = cookielib.CookieJar() cookiejar_from_dict({'foo': 'bar'}, cj) - s = requests.session() s.cookies = cj r = s.get(httpbin('cookies')) # Make sure the cookie was sent @@ -381,46 +493,69 @@ class TestRequests: # Make sure the session cj is still the custom one assert s.cookies is cj - def test_param_cookiejar_works(self, httpbin): + def test_param_cookiejar_works(self, s, httpbin): cj = cookielib.CookieJar() cookiejar_from_dict({'foo': 'bar'}, cj) - s = requests.session() r = s.get(httpbin('cookies'), cookies=cj) # Make sure the cookie was sent assert r.json()['cookies']['foo'] == 'bar' - def test_cookielib_cookiejar_on_redirect(self, httpbin): + def test_cookielib_cookiejar_on_redirect(self, s, httpbin): """Tests resolve_redirect doesn't fail when merging cookies with non-RequestsCookieJar cookiejar. See GH #3579 """ cj = cookiejar_from_dict({'foo': 'bar'}, cookielib.CookieJar()) - s = requests.Session() s.cookies = cookiejar_from_dict({'cookie': 'tasty'}) - # Prepare request without using Session req = requests.Request('GET', httpbin('headers'), cookies=cj) prep_req = req.prepare() - # Send request and simulate redirect resp = s.send(prep_req) resp.status_code = 302 resp.headers['location'] = httpbin('get') redirects = s.resolve_redirects(resp, prep_req) resp = next(redirects) - # Verify CookieJar isn't being converted to RequestsCookieJar assert isinstance(prep_req._cookies, cookielib.CookieJar) assert isinstance(resp.request._cookies, cookielib.CookieJar) - assert not isinstance(resp.request._cookies, requests.cookies.RequestsCookieJar) - + assert not isinstance( + resp.request._cookies, requests.cookies.RequestsCookieJar + ) cookies = {} for c in resp.request._cookies: cookies[c.name] = c.value assert cookies['foo'] == 'bar' assert cookies['cookie'] == 'tasty' + @pytest.mark.parametrize( + 'jar', (requests.cookies.RequestsCookieJar(), cookielib.CookieJar()) + ) + def test_custom_cookie_policy_persistence(self, s, httpbin, jar): + """Verify a custom CookiePolicy is propagated on each session request.""" + + class TestCookiePolicy(cookielib.DefaultCookiePolicy): + """Policy to restrict all cookies from localhost (127.0.0.1).""" + + def __init__(self): + cookielib.DefaultCookiePolicy.__init__( + self, blocked_domains=['127.0.0.1'] + ) + + # Establish session with jar and set some cookies. + s.cookies = jar + s.get(httpbin('cookies/set?k1=v1&k2=v2')) + assert len(s.cookies) == 2 + # Set different policy. + s.cookies.set_policy(TestCookiePolicy()) + assert isinstance(s.cookies._policy, TestCookiePolicy) + # No cookies were sent to our blocked domain and none were set. + resp = s.get(httpbin('cookies/set?k3=v3')) + assert 'Cookie' not in resp.request.headers + assert len(s.cookies) == 2 + assert 'k3' not in s.cookies + def test_requests_in_history_are_not_overridden(self, httpbin): resp = requests.get(httpbin('redirect/3')) urls = [r.url for r in resp.history] @@ -435,26 +570,24 @@ class TestRequests: assert isinstance(resp.history, list) assert not isinstance(resp.history, tuple) - def test_headers_on_session_with_None_are_not_sent(self, httpbin): + def test_headers_on_session_with_None_are_not_sent(self, httpbin, s): """Do not send headers in Session.headers with None values.""" - ses = requests.Session() - ses.headers['Accept-Encoding'] = None + s.headers['Accept-Encoding'] = None req = requests.Request('GET', httpbin('get')) - prep = ses.prepare_request(req) + prep = s.prepare_request(req) assert 'Accept-Encoding' not in prep.headers - def test_headers_preserve_order(self, httpbin): + def test_headers_preserve_order(self, s, httpbin): """Preserve order when headers provided as OrderedDict.""" - ses = requests.Session() - ses.headers = OrderedDict() - ses.headers['Accept-Encoding'] = 'identity' - ses.headers['First'] = '1' - ses.headers['Second'] = '2' - headers = OrderedDict([('Third', '3'), ('Fourth', '4')]) + s.headers = collections.OrderedDict() + s.headers['Accept-Encoding'] = 'identity' + s.headers['First'] = '1' + s.headers['Second'] = '2' + headers = collections.OrderedDict([('Third', '3'), ('Fourth', '4')]) headers['Fifth'] = '5' headers['Second'] = '222' req = requests.Request('GET', httpbin('get'), headers=headers) - prep = ses.prepare_request(req) + prep = s.prepare_request(req) items = list(prep.headers.items()) assert items[0] == ('Accept-Encoding', 'identity') assert items[1] == ('First', '1') @@ -465,9 +598,7 @@ class TestRequests: @pytest.mark.parametrize('key', ('User-agent', 'user-agent')) def test_user_agent_transfers(self, httpbin, key): - heads = {key: 'Mozilla/5.0 (github.com/requests/requests)'} - r = requests.get(httpbin('user-agent'), headers=heads) assert heads[key] in r.text @@ -479,56 +610,58 @@ class TestRequests: r = requests.put(httpbin('put')) assert r.status_code == 200 - def test_BASICAUTH_TUPLE_HTTP_200_OK_GET(self, httpbin): + def test_BASICAUTH_TUPLE_HTTP_200_OK_GET(self, httpbin, s): auth = ('user', 'pass') url = httpbin('basic-auth', 'user', 'pass') - r = requests.get(url, auth=auth) assert r.status_code == 200 - r = requests.get(url) assert r.status_code == 401 - - s = requests.session() s.auth = auth r = s.get(url) assert r.status_code == 200 @pytest.mark.parametrize( - 'username, password', ( + 'username, password', + ( ('user', 'pass'), (u'имя'.encode('utf-8'), u'пароль'.encode('utf-8')), - (42, 42), - (None, None), - )) + ), + ) def test_set_basicauth(self, httpbin, username, password): auth = (username, password) url = httpbin('get') - r = requests.Request('GET', url, auth=auth) p = r.prepare() - - assert p.headers['Authorization'] == _basic_auth_str(username, password) - - def test_basicauth_encodes_byte_strings(self): - """Ensure b'test' formats as the byte string "test" rather - than the unicode string "b'test'" in Python 3. - """ - auth = (b'\xc5\xafsername', b'test\xc6\xb6') - r = requests.Request('GET', 'http://localhost', auth=auth) - p = r.prepare() - - assert p.headers['Authorization'] == 'Basic xa9zZXJuYW1lOnRlc3TGtg==' + assert p.headers['Authorization'] == _basic_auth_str( + username, password + ) @pytest.mark.parametrize( - 'url, exception', ( - # Connecting to an unknown domain should raise a ConnectionError - ('http://doesnotexist.google.com', ConnectionError), - # Connecting to an invalid port should raise a ConnectionError - ('http://localhost:1', ConnectionError), - # Inputing a URL that cannot be parsed should raise an InvalidURL error - ('http://fe80::5054:ff:fe5a:fc0', InvalidURL) - )) + 'username, password', (('user', 1234), (None, 'test')) + ) + def test_non_str_basicauth(self, username, password): + """Ensure we only allow string or bytes values for basicauth""" + with pytest.raises(TypeError) as e: + requests.auth._basic_auth_str(username, password) + assert 'must be of type str or bytes' in str(e) + + + # def test_basicauth_encodes_byte_strings(self): + # """Ensure b'test' formats as the byte string "test" rather + # than the unicode string "b'test'" in Python 3. + # """ + # auth = (b'\xc5\xafsername', b'test\xc6\xb6') + # r = requests.Request('GET', 'http://localhost', auth=auth) + # p = r.prepare() + # assert p.headers['Authorization'] == 'Basic xa9zZXJuYW1lOnRlc3TGtg==' + @pytest.mark.parametrize( + 'url, exception', + (('http://doesnotexist.google.com', ConnectionError), ('http://localhost:1', ConnectionError), ('http://fe80::5054:ff:fe5a:fc0', InvalidURL)), + # Connecting to an unknown domain should raise a ConnectionError + # Connecting to an invalid port should raise a ConnectionError + # Inputing a URL that cannot be parsed should raise an InvalidURL error + ) def test_errors(self, url, exception): with pytest.raises(exception): requests.get(url, timeout=1) @@ -536,47 +669,32 @@ class TestRequests: def test_proxy_error(self): # any proxy related error (address resolution, no route to host, etc) should result in a ProxyError with pytest.raises(ProxyError): - requests.get('http://localhost:1', proxies={'http': 'non-resolvable-address'}) + requests.get( + 'http://localhost:1', + proxies={'http': 'non-resolvable-address'}, + ) - def test_proxy_error_on_bad_url(self, httpbin, httpbin_secure): - with pytest.raises(InvalidProxyURL): - requests.get(httpbin_secure(), proxies={'https': 'http:/badproxyurl:3128'}) - - with pytest.raises(InvalidProxyURL): - requests.get(httpbin(), proxies={'http': 'http://:8080'}) - - with pytest.raises(InvalidProxyURL): - requests.get(httpbin_secure(), proxies={'https': 'https://'}) - - with pytest.raises(InvalidProxyURL): - requests.get(httpbin(), proxies={'http': 'http:///example.com:8080'}) - - def test_basicauth_with_netrc(self, httpbin): + def test_basicauth_with_netrc(self, httpbin, s): auth = ('user', 'pass') wrong_auth = ('wronguser', 'wrongpass') url = httpbin('basic-auth', 'user', 'pass') - old_auth = requests.sessions.get_netrc_auth - try: + def get_netrc_auth_mock(url): return auth - requests.sessions.get_netrc_auth = get_netrc_auth_mock + requests.sessions.get_netrc_auth = get_netrc_auth_mock # Should use netrc and work. r = requests.get(url) assert r.status_code == 200 - # Given auth should override and fail. r = requests.get(url, auth=wrong_auth) assert r.status_code == 401 - s = requests.session() - # Should use netrc and work. r = s.get(url) assert r.status_code == 200 - # Given auth should override and fail. s.auth = wrong_auth r = s.get(url) @@ -584,97 +702,68 @@ class TestRequests: finally: requests.sessions.get_netrc_auth = old_auth - def test_DIGEST_HTTP_200_OK_GET(self, httpbin): + def test_DIGEST_HTTP_200_OK_GET(self, httpbin, s): + auth = HTTPDigestAuth('user', 'pass') + url = httpbin('digest-auth', 'auth', 'user', 'pass') + r = requests.get(url, auth=auth) + assert r.status_code == 200 + r = requests.get(url) + assert r.status_code == 401 - for authtype in self.digest_auth_algo: - auth = HTTPDigestAuth('user', 'pass') - url = httpbin('digest-auth', 'auth', 'user', 'pass', authtype, 'never') - - r = requests.get(url, auth=auth) - assert r.status_code == 200 - - r = requests.get(url) - assert r.status_code == 401 - print(r.headers['WWW-Authenticate']) - - s = requests.session() - s.auth = HTTPDigestAuth('user', 'pass') - r = s.get(url) - assert r.status_code == 200 + s.auth = HTTPDigestAuth('user', 'pass') + r = s.get(url) + assert r.status_code == 200 def test_DIGEST_AUTH_RETURNS_COOKIE(self, httpbin): + url = httpbin('digest-auth', 'auth', 'user', 'pass') + auth = HTTPDigestAuth('user', 'pass') + r = requests.get(url) + assert r.cookies['fake'] == 'fake_value' + r = requests.get(url, auth=auth) + assert r.status_code == 200 - for authtype in self.digest_auth_algo: - url = httpbin('digest-auth', 'auth', 'user', 'pass', authtype) - auth = HTTPDigestAuth('user', 'pass') - r = requests.get(url) - assert r.cookies['fake'] == 'fake_value' + def test_DIGEST_AUTH_SETS_SESSION_COOKIES(self, httpbin, s): + url = httpbin('digest-auth', 'auth', 'user', 'pass') + auth = HTTPDigestAuth('user', 'pass') - r = requests.get(url, auth=auth) - assert r.status_code == 200 - - def test_DIGEST_AUTH_SETS_SESSION_COOKIES(self, httpbin): - - for authtype in self.digest_auth_algo: - url = httpbin('digest-auth', 'auth', 'user', 'pass', authtype) - auth = HTTPDigestAuth('user', 'pass') - s = requests.Session() - s.get(url, auth=auth) - assert s.cookies['fake'] == 'fake_value' + s.get(url, auth=auth) + assert s.cookies['fake'] == 'fake_value' def test_DIGEST_STREAM(self, httpbin): + auth = HTTPDigestAuth('user', 'pass') + url = httpbin('digest-auth', 'auth', 'user', 'pass') + r = requests.get(url, auth=auth, stream=True) + assert r.raw.read() != b'' + r = requests.get(url, auth=auth, stream=False) + assert r.raw.read() == b'' - for authtype in self.digest_auth_algo: - auth = HTTPDigestAuth('user', 'pass') - url = httpbin('digest-auth', 'auth', 'user', 'pass', authtype) - - r = requests.get(url, auth=auth, stream=True) - assert r.raw.read() != b'' - - r = requests.get(url, auth=auth, stream=False) - assert r.raw.read() == b'' - - def test_DIGESTAUTH_WRONG_HTTP_401_GET(self, httpbin): - - for authtype in self.digest_auth_algo: - auth = HTTPDigestAuth('user', 'wrongpass') - url = httpbin('digest-auth', 'auth', 'user', 'pass', authtype) - - r = requests.get(url, auth=auth) - assert r.status_code == 401 - - r = requests.get(url) - assert r.status_code == 401 - - s = requests.session() - s.auth = auth - r = s.get(url) - assert r.status_code == 401 + def test_DIGESTAUTH_WRONG_HTTP_401_GET(self, httpbin, s): + auth = HTTPDigestAuth('user', 'wrongpass') + url = httpbin('digest-auth', 'auth', 'user', 'pass') + r = requests.get(url, auth=auth) + assert r.status_code == 401 + r = requests.get(url) + assert r.status_code == 401 + s.auth = auth + r = s.get(url) + assert r.status_code == 401 def test_DIGESTAUTH_QUOTES_QOP_VALUE(self, httpbin): - - for authtype in self.digest_auth_algo: - auth = HTTPDigestAuth('user', 'pass') - url = httpbin('digest-auth', 'auth', 'user', 'pass', authtype) - - r = requests.get(url, auth=auth) - assert '"auth"' in r.request.headers['Authorization'] + auth = HTTPDigestAuth('user', 'pass') + url = httpbin('digest-auth', 'auth', 'user', 'pass') + r = requests.get(url, auth=auth) + assert '"auth"' in r.request.headers['Authorization'] def test_POSTBIN_GET_POST_FILES(self, httpbin): - url = httpbin('post') requests.post(url).raise_for_status() - post1 = requests.post(url, data={'some': 'data'}) assert post1.status_code == 200 - with open('Pipfile') as f: post2 = requests.post(url, files={'some': f}) assert post2.status_code == 200 - post4 = requests.post(url, data='[{"some": "json"}]') assert post4.status_code == 200 - with pytest.raises(ValueError): requests.post(url, files=['bad file data']) @@ -689,6 +778,7 @@ class TestRequests: def test_POSTBIN_SEEKED_OBJECT_WITH_NO_ITER(self, httpbin): class TestStream(object): + def __init__(self, data): self.data = data.encode() self.length = len(self.data) @@ -699,7 +789,7 @@ class TestRequests: def read(self, size=None): if size: - ret = self.data[self.index:self.index + size] + ret = self.data[self.index: self.index + size] self.index += size else: ret = self.data[self.index:] @@ -721,7 +811,6 @@ class TestRequests: post1 = requests.post(httpbin('post'), data=test) assert post1.status_code == 200 assert post1.json()['data'] == 'test' - test = TestStream('test') test.seek(2) post2 = requests.post(httpbin('post'), data=test) @@ -729,25 +818,24 @@ class TestRequests: assert post2.json()['data'] == 'st' def test_POSTBIN_GET_POST_FILES_WITH_DATA(self, httpbin): - url = httpbin('post') requests.post(url).raise_for_status() - post1 = requests.post(url, data={'some': 'data'}) assert post1.status_code == 200 - with open('Pipfile') as f: - post2 = requests.post(url, data={'some': 'data'}, files={'some': f}) + post2 = requests.post( + url, data={'some': 'data'}, files={'some': f} + ) assert post2.status_code == 200 - post4 = requests.post(url, data='[{"some": "json"}]') assert post4.status_code == 200 - with pytest.raises(ValueError): requests.post(url, files=['bad file data']) def test_post_with_custom_mapping(self, httpbin): - class CustomMapping(MutableMapping): + + class CustomMapping(collections.MutableMapping): + def __init__(self, *args, **kwargs): self.data = dict(*args, **kwargs) @@ -774,8 +862,14 @@ class TestRequests: def test_conflicting_post_params(self, httpbin): url = httpbin('post') with open('Pipfile') as f: - pytest.raises(ValueError, "requests.post(url, data='[{\"some\": \"data\"}]', files={'some': f})") - pytest.raises(ValueError, "requests.post(url, data=u('[{\"some\": \"data\"}]'), files={'some': f})") + pytest.raises( + ValueError, + "requests.post(url, data='[{\"some\": \"data\"}]', files={'some': f})", + ) + pytest.raises( + ValueError, + "requests.post(url, data=u('[{\"some\": \"data\"}]'), files={'some': f})", + ) def test_request_ok_set(self, httpbin): r = requests.get(httpbin('status', '404')) @@ -785,22 +879,27 @@ class TestRequests: r = requests.get(httpbin('status', '404')) with pytest.raises(requests.exceptions.HTTPError): r.raise_for_status() - r = requests.get(httpbin('status', '500')) assert not r.ok + def test_raise_for_status_returns_self(self, httpbin): + r = requests.get(httpbin('status', '200')) + assert r.raise_for_status() is r + def test_decompress_gzip(self, httpbin): r = requests.get(httpbin('gzip')) r.content.decode('ascii') @pytest.mark.parametrize( - 'url, params', ( + 'url, params', + ( ('/get', {'foo': 'føø'}), ('/get', {'føø': 'føø'}), ('/get', {'føø': 'føø'}), ('/get', {'foo': 'foo'}), ('ø', {'foo': 'foo'}), - )) + ), + ) def test_unicode_get(self, httpbin, url, params): requests.get(httpbin(url), params=params) @@ -808,7 +907,8 @@ class TestRequests: requests.put( httpbin('put'), headers={str('Content-Type'): 'application/octet-stream'}, - data='\xff') # compat.str is unicode. + data='\xff', + ) # compat.str is unicode. def test_pyopenssl_redirect(self, httpbin_secure, httpbin_ca_bundle): requests.get(httpbin_secure('status', '301'), verify=httpbin_ca_bundle) @@ -817,17 +917,28 @@ class TestRequests: INVALID_PATH = '/garbage' with pytest.raises(IOError) as e: requests.get(httpbin_secure(), verify=INVALID_PATH) - assert str(e.value) == 'Could not find a suitable TLS CA certificate bundle, invalid path: {}'.format(INVALID_PATH) + assert str( + e.value + ) == 'Could not find a suitable TLS CA certificate bundle, invalid path: {0}'.format( + INVALID_PATH + ) def test_invalid_ssl_certificate_files(self, httpbin_secure): INVALID_PATH = '/garbage' with pytest.raises(IOError) as e: requests.get(httpbin_secure(), cert=INVALID_PATH) - assert str(e.value) == 'Could not find the TLS certificate file, invalid path: {}'.format(INVALID_PATH) - + assert str( + e.value + ) == 'Could not find the TLS certificate file, invalid path: {0}'.format( + INVALID_PATH + ) with pytest.raises(IOError) as e: requests.get(httpbin_secure(), cert=('.', INVALID_PATH)) - assert str(e.value) == 'Could not find the TLS key file, invalid path: {}'.format(INVALID_PATH) + assert str( + e.value + ) == 'Could not find the TLS key file, invalid path: {0}'.format( + INVALID_PATH + ) def test_http_with_certificate(self, httpbin): r = requests.get(httpbin(), cert='.') @@ -836,22 +947,26 @@ class TestRequests: def test_https_warnings(self, httpbin_secure, httpbin_ca_bundle): """warnings are emitted with requests.get""" if HAS_MODERN_SSL or HAS_PYOPENSSL: - warnings_expected = ('SubjectAltNameWarning', ) + warnings_expected = ('SubjectAltNameWarning',) else: - warnings_expected = ('SNIMissingWarning', - 'InsecurePlatformWarning', - 'SubjectAltNameWarning', ) - + warnings_expected = ( + 'SNIMissingWarning', + 'InsecurePlatformWarning', + 'SubjectAltNameWarning', + ) with pytest.warns(None) as warning_records: warnings.simplefilter('always') - requests.get(httpbin_secure('status', '200'), - verify=httpbin_ca_bundle) - - warning_records = [item for item in warning_records - if item.category.__name__ != 'ResourceWarning'] - + requests.get( + httpbin_secure('status', '200'), verify=httpbin_ca_bundle + ) + warning_records = [ + item + for item in warning_records + if item.category.__name__ != 'ResourceWarning' + ] warnings_category = tuple( - item.category.__name__ for item in warning_records) + item.category.__name__ for item in warning_records + ) assert warnings_category == warnings_expected def test_certificate_failure(self, httpbin_secure): @@ -864,7 +979,6 @@ class TestRequests: requests.get(httpbin_secure('status', '200')) def test_urlencoded_get_query_multivalued_param(self, httpbin): - r = requests.get(httpbin('get'), params={'test': ['foo', 'baz']}) assert r.status_code == 200 assert r.url == httpbin('get?test=foo&test=baz') @@ -876,31 +990,39 @@ class TestRequests: assert prep.body == 'test=foo&test=baz' def test_different_encodings_dont_break_post(self, httpbin): - r = requests.post(httpbin('post'), + r = requests.post( + httpbin('post'), data={'stuff': json.dumps({'a': 123})}, params={'blah': 'asdf1234'}, - files={'file': ('test_requests.py', open(__file__, 'rb'))}) + files={'file': ('test_requests.py', open(__file__, 'rb'))}, + ) assert r.status_code == 200 @pytest.mark.parametrize( - 'data', ( + 'data', + ( {'stuff': u('ëlïxr')}, {'stuff': u('ëlïxr').encode('utf-8')}, {'stuff': 'elixr'}, {'stuff': 'elixr'.encode('utf-8')}, - )) + ), + ) def test_unicode_multipart_post(self, httpbin, data): - r = requests.post(httpbin('post'), + r = requests.post( + httpbin('post'), data=data, - files={'file': ('test_requests.py', open(__file__, 'rb'))}) + files={'file': ('test_requests.py', open(__file__, 'rb'))}, + ) assert r.status_code == 200 def test_unicode_multipart_post_fieldnames(self, httpbin): filename = os.path.splitext(__file__)[0] + '.py' r = requests.Request( - method='POST', url=httpbin('post'), + method='POST', + url=httpbin('post'), data={'stuff'.encode('utf-8'): 'elixr'}, - files={'file': ('test_requests.py', open(filename, 'rb'))}) + files={'file': ('test_requests.py', open(filename, 'rb'))}, + ) prep = r.prepare() assert b'name="stuff"' in prep.body assert b'name="b\'stuff\'"' not in prep.body @@ -908,24 +1030,22 @@ class TestRequests: def test_unicode_method_name(self, httpbin): files = {'file': open(__file__, 'rb')} r = requests.request( - method=u('POST'), url=httpbin('post'), files=files) + method=u('POST'), url=httpbin('post'), files=files + ) assert r.status_code == 200 - def test_unicode_method_name_with_request_object(self, httpbin): + def test_unicode_method_name_with_request_object(self, httpbin, s): files = {'file': open(__file__, 'rb')} - s = requests.Session() + req = requests.Request(u('POST'), httpbin('post'), files=files) prep = s.prepare_request(req) assert isinstance(prep.method, builtin_str) assert prep.method == 'POST' - resp = s.send(prep) assert resp.status_code == 200 - def test_non_prepared_request_error(self): - s = requests.Session() + def test_non_prepared_request_error(self, s): req = requests.Request(u('POST'), '/') - with pytest.raises(ValueError) as e: s.send(req) assert str(e.value) == 'You can only send PreparedRequests.' @@ -936,56 +1056,61 @@ class TestRequests: data={'stuff': json.dumps({'a': 123})}, files={ 'file1': ('test_requests.py', open(__file__, 'rb')), - 'file2': ('test_requests', open(__file__, 'rb'), - 'text/py-content-type')}) + 'file2': ( + 'test_requests', + open(__file__, 'rb'), + 'text/py-content-type', + ), + }, + ) assert r.status_code == 200 assert b"text/py-content-type" in r.request.body - def test_hook_receives_request_arguments(self, httpbin): + def test_hook_receives_request_arguments(self, httpbin, s): + def hook(resp, **kwargs): assert resp is not None assert kwargs != {} - s = requests.Session() r = requests.Request('GET', httpbin(), hooks={'response': hook}) prep = s.prepare_request(r) s.send(prep) - def test_session_hooks_are_used_with_no_request_hooks(self, httpbin): + def test_session_hooks_are_used_with_no_request_hooks(self, httpbin, s): hook = lambda x, *args, **kwargs: x - s = requests.Session() + s.hooks['response'].append(hook) r = requests.Request('GET', httpbin()) prep = s.prepare_request(r) assert prep.hooks['response'] != [] assert prep.hooks['response'] == [hook] - def test_session_hooks_are_overridden_by_request_hooks(self, httpbin): + def test_session_hooks_are_overridden_by_request_hooks(self, httpbin, s): hook1 = lambda x, *args, **kwargs: x hook2 = lambda x, *args, **kwargs: x assert hook1 is not hook2 - s = requests.Session() + s.hooks['response'].append(hook2) r = requests.Request('GET', httpbin(), hooks={'response': [hook1]}) prep = s.prepare_request(r) assert prep.hooks['response'] == [hook1] - def test_prepared_request_hook(self, httpbin): + def test_prepared_request_hook(self, httpbin, s): + def hook(resp, **kwargs): - resp.hook_working = True + resp.headers['hook-working'] = 'True' return resp req = requests.Request('GET', httpbin(), hooks={'response': hook}) prep = req.prepare() - - s = requests.Session() s.proxies = getproxies() resp = s.send(prep) + assert resp.headers['hook-working'] - assert hasattr(resp, 'hook_working') + def test_prepared_from_session(self, httpbin, s): - def test_prepared_from_session(self, httpbin): class DummyAuth(requests.auth.AuthBase): + def __call__(self, r): r.headers['Dummy-Auth-Test'] = 'dummy-auth-test-ok' return r @@ -993,31 +1118,28 @@ class TestRequests: req = requests.Request('GET', httpbin('headers')) assert not req.auth - s = requests.Session() s.auth = DummyAuth() - prep = s.prepare_request(req) resp = s.send(prep) - assert resp.json()['headers'][ - 'Dummy-Auth-Test'] == 'dummy-auth-test-ok' + 'Dummy-Auth-Test' + ] == 'dummy-auth-test-ok' - def test_prepare_request_with_bytestring_url(self): - req = requests.Request('GET', b'https://httpbin.org/') - s = requests.Session() - prep = s.prepare_request(req) - assert prep.url == "https://httpbin.org/" - - def test_request_with_bytestring_host(self, httpbin): - s = requests.Session() - resp = s.request( - 'GET', - httpbin('cookies/set?cookie=value'), - allow_redirects=False, - headers={'Host': b'httpbin.org'} - ) - assert resp.cookies.get('cookie') == 'value' + # def test_prepare_request_with_bytestring_url(self): + # req = requests.Request('GET', b'https://httpbin.org/') + # s = requests.Session() + # prep = s.prepare_request(req) + # assert prep.url == "https://httpbin.org/" + # def test_request_with_bytestring_host(self, httpbin): + # s = requests.Session() + # resp = s.request( + # 'GET', + # httpbin('cookies/set?cookie=value'), + # allow_redirects=False, + # headers={'Host': b'httpbin.org'}, + # ) + # assert resp.cookies.get('cookie') == 'value' def test_links(self): r = requests.Response() r.headers = { @@ -1028,17 +1150,19 @@ class TestRequests: 'date': 'Sat, 26 Jan 2013 16:47:56 GMT', 'etag': '"6ff6a73c0e446c1f61614769e3ceb778"', 'last-modified': 'Sat, 26 Jan 2013 16:22:39 GMT', - 'link': ('; rel="next", ; ' - ' rel="last"'), + 'link': ( + '; rel="next", ; ' + ' rel="last"' + ), 'server': 'GitHub.com', 'status': '200 OK', 'vary': 'Accept', 'x-content-type-options': 'nosniff', 'x-github-media-type': 'github.beta', 'x-ratelimit-limit': '60', - 'x-ratelimit-remaining': '57' + 'x-ratelimit-remaining': '57', } assert r.links['next']['rel'] == 'next' @@ -1048,13 +1172,10 @@ class TestRequests: secure = True domain = 'test.com' rest = {'HttpOnly': True} - jar = requests.cookies.RequestsCookieJar() jar.set(key, value, secure=secure, domain=domain, rest=rest) - assert len(jar) == 1 assert 'some_cookie' in jar - cookie = list(jar)[0] assert cookie.secure == secure assert cookie.domain == domain @@ -1063,18 +1184,14 @@ class TestRequests: def test_cookie_as_dict_keeps_len(self): key = 'some_cookie' value = 'some_value' - key1 = 'some_cookie1' value1 = 'some_value1' - jar = requests.cookies.RequestsCookieJar() jar.set(key, value) jar.set(key1, value1) - d1 = dict(jar) d2 = dict(jar.iteritems()) d3 = dict(jar.items()) - assert len(jar) == 2 assert len(d1) == 2 assert len(d2) == 2 @@ -1083,18 +1200,14 @@ class TestRequests: def test_cookie_as_dict_keeps_items(self): key = 'some_cookie' value = 'some_value' - key1 = 'some_cookie1' value1 = 'some_value1' - jar = requests.cookies.RequestsCookieJar() jar.set(key, value) jar.set(key1, value1) - d1 = dict(jar) d2 = dict(jar.iteritems()) d3 = dict(jar.items()) - assert d1['some_cookie'] == 'some_value' assert d2['some_cookie'] == 'some_value' assert d3['some_cookie1'] == 'some_value1' @@ -1102,14 +1215,11 @@ class TestRequests: def test_cookie_as_dict_keys(self): key = 'some_cookie' value = 'some_value' - key1 = 'some_cookie1' value1 = 'some_value1' - jar = requests.cookies.RequestsCookieJar() jar.set(key, value) jar.set(key1, value1) - keys = jar.keys() assert keys == list(keys) # make sure one can use keys multiple times @@ -1118,14 +1228,11 @@ class TestRequests: def test_cookie_as_dict_values(self): key = 'some_cookie' value = 'some_value' - key1 = 'some_cookie1' value1 = 'some_value1' - jar = requests.cookies.RequestsCookieJar() jar.set(key, value) jar.set(key1, value1) - values = jar.values() assert values == list(values) # make sure one can use values multiple times @@ -1134,14 +1241,11 @@ class TestRequests: def test_cookie_as_dict_items(self): key = 'some_cookie' value = 'some_value' - key1 = 'some_cookie1' value1 = 'some_value1' - jar = requests.cookies.RequestsCookieJar() jar.set(key, value) jar.set(key1, value1) - items = jar.items() assert items == list(items) # make sure one can use items multiple times @@ -1152,18 +1256,15 @@ class TestRequests: value = 'some_value' domain1 = 'test1.com' domain2 = 'test2.com' - jar = requests.cookies.RequestsCookieJar() jar.set(key, value, domain=domain1) jar.set(key, value, domain=domain2) assert key in jar items = jar.items() assert len(items) == 2 - # Verify that CookieConflictError is raised if domain is not specified with pytest.raises(requests.cookies.CookieConflictError): jar.get(key) - # Verify that CookieConflictError is not raised if domain is specified cookie = jar.get(key, domain=domain1) assert cookie == value @@ -1172,7 +1273,6 @@ class TestRequests: key = 'some_cookie' value = 'some_value' path = 'some_path' - jar = requests.cookies.RequestsCookieJar() jar.set(key, value, path=path) jar.set(key, value) @@ -1190,7 +1290,11 @@ class TestRequests: def test_time_elapsed_blank(self, httpbin): r = requests.get(httpbin('get')) td = r.elapsed - total_seconds = ((td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6) + total_seconds = ( + (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10 ** 6) / + 10 ** + 6 + ) assert total_seconds > 0.0 def test_empty_response_has_content_none(self): @@ -1204,6 +1308,7 @@ class TestRequests: def read_mock(amt, decode_content=None): return read_(amt) + setattr(io, 'read', read_mock) r.raw = io assert next(iter(r)) @@ -1217,10 +1322,8 @@ class TestRequests: r._content_consumed = True r._content = b'the content' r.encoding = 'ascii' - chunks = r.iter_content(decode_unicode=True) assert all(isinstance(chunk, str) for chunk in chunks) - # also for streaming r = requests.Response() r.raw = io.BytesIO(b'the content') @@ -1228,6 +1331,18 @@ class TestRequests: chunks = r.iter_content(decode_unicode=True) assert all(isinstance(chunk, str) for chunk in chunks) + @pytest.mark.parametrize( + 'encoding, exception', + ((None, TypeError), ('invalid encoding', LookupError)), + ) + def test_decode_unicode_encoding(self, encoding, exception): + # raise an exception if encoding isn't set + r = requests.Response() + r.raw = io.BytesIO(b'the content') + r.encoding = encoding + with pytest.raises(exception): + chunks = r.iter_content(decode_unicode=True) + def test_response_reason_unicode(self): # check for unicode HTTP status r = requests.Response() @@ -1257,12 +1372,10 @@ class TestRequests: r.raw = io.BytesIO(b'the content') chunks = r.iter_content(1) assert all(len(chunk) == 1 for chunk in chunks) - r = requests.Response() r.raw = io.BytesIO(b'the content') chunks = r.iter_content(None) assert list(chunks) == [b'the content'] - r = requests.Response() r.raw = io.BytesIO(b'the content') with pytest.raises(TypeError): @@ -1270,59 +1383,153 @@ class TestRequests: def test_request_and_response_are_pickleable(self, httpbin): r = requests.get(httpbin('get')) - # verify we can pickle the original request assert pickle.loads(pickle.dumps(r.request)) - # verify we can pickle the response and that we have access to # the original request. pr = pickle.loads(pickle.dumps(r)) assert r.request.url == pr.request.url assert r.request.headers == pr.request.headers - def test_prepared_request_is_pickleable(self, httpbin): - p = requests.Request('GET', httpbin('get')).prepare() + @pytest.mark.skip(reason="TODO: Doesn't work with __slots__.") + def test_response_lines(self): + """ + iter_lines should be able to handle data dribbling in which delimiters + might not be lined up ideally. + """ + mock_chunks = [ + b'This \r\n', + b'', + b'is\r', + b'\n', + b'a', + b' ', + b'', + b'', + b'test.', + b'\r', + b'\n', + b'end.', + ] + mock_data = b''.join(mock_chunks) + unicode_mock_data = mock_data.decode('utf-8') + def mock_iter_content(*args, **kwargs): + if kwargs.get("decode_unicode"): + return (e.decode('utf-8') for e in mock_chunks) + + return (e for e in mock_chunks) + + r = requests.Response() + r._content_consumed = True + r.iter_content = mock_iter_content + # decode_unicode=None, output raw bytes + assert list(r.iter_lines(delimiter=b'\r\n')) == mock_data.split( + b'\r\n' + ) + # decode_unicode=True, output unicode strings + assert list( + r.iter_lines(decode_unicode=True, delimiter=u'\r\n') + ) == unicode_mock_data.split( + u'\r\n' + ) + # When delimiter is None, we should yield the same result as splitlines() + # which supports the universal newline. + # '\r', '\n', and '\r\n' are all treated as one line break. + # decode_unicode=None, output raw bytes + result = list(r.iter_lines()) + assert result == mock_data.splitlines() + # decode_unicode=True, output unicode strings + result = list(r.iter_lines(decode_unicode=True)) + assert result == unicode_mock_data.splitlines() + # If we change all the line breaks to `\r`, we should be okay. + # decode_unicode=None, output raw bytes + mock_chunks = [chunk.replace(b'\n', b'\r') for chunk in mock_chunks] + mock_data = b''.join(mock_chunks) + assert list(r.iter_lines()) == mock_data.splitlines() + # decode_unicode=True, output unicode strings + unicode_mock_data = mock_data.decode('utf-8') + assert list( + r.iter_lines(decode_unicode=True) + ) == unicode_mock_data.splitlines( + ) + + @pytest.mark.parametrize( + 'content, expected_no_delimiter, expected_delimiter', + (([b''], [], []), ([b'line\n'], [u'line'], [u'line\n']), ([b'line', b'\n'], [u'line'], [u'line\n']), ([b'line\r\n'], [u'line'], [u'line', u'']), ([b'line\r\n', b''], [u'line'], [u'line', u'']), ([b'line', b'\r\n'], [u'line'], [u'line', u'']), ([b'a\r', b'\nb\r'], [u'a', u'b'], [u'a', u'b\r']), ([b'a\r', b'\n', b'\nb'], [u'a', u'', u'b'], [u'a', u'\nb']), ([b'a\n', b'\nb'], [u'a', u'', u'b'], [u'a\n\nb']), ([b'a\r\n', b'\rb\n'], [u'a', u'', u'b'], [u'a', u'\rb\n']), ([b'a\nb', b'c'], [u'a', u'bc'], [u'a\nbc']), ([b'a\n', b'\rb', b'\r\nc'], [u'a', u'', u'b', u'c'], [u'a\n\rb', u'c']), ([b'a\r\nb', b'', b'c'], [u'a', u'bc'], [u'a', u'bc'])), + # Empty chunk in the end of stream, same behavior as the previous # Empty chunk with pending data + ) + @pytest.mark.skip(reason="TODO: Doesn't work with __slots__") + def test_response_lines_parametrized( + self, content, expected_no_delimiter, expected_delimiter + ): + """ + Test a lot of potential chunk splits to ensure consistency of + iter_lines(delimiter=x), as well as the legacy behavior of + iter_lines() without delimiter + https://github.com/kennethreitz/requests/pull/2431#issuecomment-72333964 + """ + mock_chunks = content + + def mock_iter_content(*args, **kwargs): + if kwargs.get("decode_unicode"): + return (e.decode('utf-8') for e in mock_chunks) + + return (e for e in mock_chunks) + + r = requests.Response() + r._content_consumed = True + r.iter_content = mock_iter_content + # decode_unicode=True, output unicode strings + assert list(r.iter_lines(decode_unicode=True)) == expected_no_delimiter + assert list( + r.iter_lines(decode_unicode=True, delimiter='\r\n') + ) == expected_delimiter + # decode_unicode=None, output raw bytes + assert list(r.iter_lines()) == [ + line.encode('utf-8') for line in expected_no_delimiter + ] + assert list(r.iter_lines(delimiter=b'\r\n')) == [ + line.encode('utf-8') for line in expected_delimiter + ] + + def test_prepared_request_is_pickleable(self, httpbin, s): + p = requests.Request('GET', httpbin('get')).prepare() # Verify PreparedRequest can be pickled and unpickled r = pickle.loads(pickle.dumps(p)) assert r.url == p.url assert r.headers == p.headers assert r.body == p.body - # Verify unpickled PreparedRequest sends properly - s = requests.Session() + resp = s.send(r) assert resp.status_code == 200 - def test_prepared_request_with_file_is_pickleable(self, httpbin): + def test_prepared_request_with_file_is_pickleable(self, httpbin, s): files = {'file': open(__file__, 'rb')} r = requests.Request('POST', httpbin('post'), files=files) p = r.prepare() - # Verify PreparedRequest can be pickled and unpickled r = pickle.loads(pickle.dumps(p)) assert r.url == p.url assert r.headers == p.headers assert r.body == p.body - # Verify unpickled PreparedRequest sends properly - s = requests.Session() + resp = s.send(r) assert resp.status_code == 200 - def test_prepared_request_with_hook_is_pickleable(self, httpbin): + def test_prepared_request_with_hook_is_pickleable(self, httpbin, s): r = requests.Request('GET', httpbin('get'), hooks=default_hooks()) p = r.prepare() - # Verify PreparedRequest can be pickled r = pickle.loads(pickle.dumps(p)) assert r.url == p.url assert r.headers == p.headers assert r.body == p.body assert r.hooks == p.hooks - # Verify unpickled PreparedRequest sends properly - s = requests.Session() + resp = s.send(r) assert resp.status_code == 200 @@ -1341,19 +1548,17 @@ class TestRequests: assert str(error) == 'message' assert error.response == response - def test_session_pickling(self, httpbin): + def test_session_pickling(self, httpbin, s): r = requests.Request('GET', httpbin('get')) - s = requests.Session() s = pickle.loads(pickle.dumps(s)) s.proxies = getproxies() - r = s.send(r.prepare()) assert r.status_code == 200 - def test_fixes_1329(self, httpbin): + def test_fixes_1329(self, httpbin, s): """Ensure that header updates are done case-insensitively.""" - s = requests.Session() + s.headers.update({'ACCEPT': 'BOGUS'}) s.headers.update({'accept': 'application/json'}) r = s.get(httpbin('get')) @@ -1369,8 +1574,8 @@ class TestRequests: assert r.status_code == 200 assert r.url.lower() == url.lower() - def test_transport_adapter_ordering(self): - s = requests.Session() + def test_transport_adapter_ordering(self, s): + order = ['https://', 'http://'] assert order == list(s.adapters) s.mount('http://git', HTTPAdapter()) @@ -1407,53 +1612,13 @@ class TestRequests: assert 'http://' in s2.adapters assert 'https://' in s2.adapters - def test_session_get_adapter_prefix_matching(self): - prefix = 'https://example.com' - more_specific_prefix = prefix + '/some/path' - - url_matching_only_prefix = prefix + '/another/path' - url_matching_more_specific_prefix = more_specific_prefix + '/longer/path' - url_not_matching_prefix = 'https://another.example.com/' - - s = requests.Session() - prefix_adapter = HTTPAdapter() - more_specific_prefix_adapter = HTTPAdapter() - s.mount(prefix, prefix_adapter) - s.mount(more_specific_prefix, more_specific_prefix_adapter) - - assert s.get_adapter(url_matching_only_prefix) is prefix_adapter - assert s.get_adapter(url_matching_more_specific_prefix) is more_specific_prefix_adapter - assert s.get_adapter(url_not_matching_prefix) not in (prefix_adapter, more_specific_prefix_adapter) - - def test_session_get_adapter_prefix_matching_mixed_case(self): - mixed_case_prefix = 'hTtPs://eXamPle.CoM/MixEd_CAse_PREfix' - url_matching_prefix = mixed_case_prefix + '/full_url' - - s = requests.Session() - my_adapter = HTTPAdapter() - s.mount(mixed_case_prefix, my_adapter) - - assert s.get_adapter(url_matching_prefix) is my_adapter - - def test_session_get_adapter_prefix_matching_is_case_insensitive(self): - mixed_case_prefix = 'hTtPs://eXamPle.CoM/MixEd_CAse_PREfix' - url_matching_prefix_with_different_case = 'HtTpS://exaMPLe.cOm/MiXeD_caSE_preFIX/another_url' - - s = requests.Session() - my_adapter = HTTPAdapter() - s.mount(mixed_case_prefix, my_adapter) - - assert s.get_adapter(url_matching_prefix_with_different_case) is my_adapter - - def test_header_remove_is_case_insensitive(self, httpbin): + def test_header_remove_is_case_insensitive(self, httpbin, s): # From issue #1321 - s = requests.Session() s.headers['foo'] = 'bar' r = s.get(httpbin('get'), headers={'FOO': None}) assert 'foo' not in r.request.headers - def test_params_are_merged_case_sensitive(self, httpbin): - s = requests.Session() + def test_params_are_merged_case_sensitive(self, httpbin, s): s.params['foo'] = 'bar' r = s.get(httpbin('get'), params={'FOO': 'bar'}) assert r.json()['args'] == {'foo': 'bar', 'FOO': 'bar'} @@ -1471,7 +1636,6 @@ class TestRequests: headers = {u('unicode'): 'blah', 'byte'.encode('ascii'): 'blah'} r = requests.Request('GET', httpbin('get'), headers=headers) p = r.prepare() - # This is testing that they are builtin strings. A bit weird, but there # we go. assert 'unicode' in p.headers.keys() @@ -1479,10 +1643,9 @@ class TestRequests: def test_header_validation(self, httpbin): """Ensure prepare_headers regex isn't flagging valid header contents.""" - headers_ok = {'foo': 'bar baz qux', - 'bar': u'fbbq'.encode('utf8'), - 'baz': '', - 'qux': '1'} + headers_ok = { + 'foo': 'bar baz qux', 'bar': 'fbbq', 'baz': '', 'qux': '1' + } r = requests.get(httpbin('get'), headers=headers_ok) assert r.request.headers['foo'] == headers_ok['foo'] @@ -1493,7 +1656,6 @@ class TestRequests: headers_int = {'foo': 3} headers_dict = {'bar': {'foo': 'bar'}} headers_list = {'baz': ['foo', 'bar']} - # Test for int with pytest.raises(InvalidHeader) as excinfo: r = requests.get(httpbin('get'), headers=headers_int) @@ -1514,7 +1676,6 @@ class TestRequests: headers_ret = {'foo': 'bar\r\nbaz: qux'} headers_lf = {'foo': 'bar\nbaz: qux'} headers_cr = {'foo': 'bar\rbaz: qux'} - # Test for newline with pytest.raises(InvalidHeader): r = requests.get(httpbin('get'), headers=headers_ret) @@ -1531,7 +1692,6 @@ class TestRequests: """ headers_space = {'foo': ' bar'} headers_tab = {'foo': ' bar'} - # Test for whitespace with pytest.raises(InvalidHeader): r = requests.get(httpbin('get'), headers=headers_space) @@ -1552,7 +1712,6 @@ class TestRequests: f.name = 2 r = requests.Request('POST', httpbin('post'), files={'f': f}) p = r.prepare() - assert 'multipart/form-data' in p.headers['Content-Type'] def test_autoset_header_values_are_native(self, httpbin): @@ -1560,7 +1719,6 @@ class TestRequests: length = '16' req = requests.Request('POST', httpbin('post'), data=data) p = req.prepare() - assert p.headers['Content-Length'] == length def test_nonhttp_schemes_dont_check_URLs(self): @@ -1588,53 +1746,19 @@ class TestRequests: r = requests.get(httpbin('redirect/1'), auth=('user', 'pass')) h1 = r.history[0].request.headers['Authorization'] h2 = r.request.headers['Authorization'] - assert h1 == h2 - def test_should_strip_auth_host_change(self): - s = requests.Session() - assert s.should_strip_auth('http://example.com/foo', 'http://another.example.com/') + def test_manual_redirect_with_partial_body_read(self, httpbin, s): - def test_should_strip_auth_http_downgrade(self): - s = requests.Session() - assert s.should_strip_auth('https://example.com/foo', 'http://example.com/bar') - - def test_should_strip_auth_https_upgrade(self): - s = requests.Session() - assert not s.should_strip_auth('http://example.com/foo', 'https://example.com/bar') - assert not s.should_strip_auth('http://example.com:80/foo', 'https://example.com/bar') - assert not s.should_strip_auth('http://example.com/foo', 'https://example.com:443/bar') - # Non-standard ports should trigger stripping - assert s.should_strip_auth('http://example.com:8080/foo', 'https://example.com/bar') - assert s.should_strip_auth('http://example.com/foo', 'https://example.com:8443/bar') - - def test_should_strip_auth_port_change(self): - s = requests.Session() - assert s.should_strip_auth('http://example.com:1234/foo', 'https://example.com:4321/bar') - - @pytest.mark.parametrize( - 'old_uri, new_uri', ( - ('https://example.com:443/foo', 'https://example.com/bar'), - ('http://example.com:80/foo', 'http://example.com/bar'), - ('https://example.com/foo', 'https://example.com:443/bar'), - ('http://example.com/foo', 'http://example.com:80/bar') - )) - def test_should_strip_auth_default_port(self, old_uri, new_uri): - s = requests.Session() - assert not s.should_strip_auth(old_uri, new_uri) - - def test_manual_redirect_with_partial_body_read(self, httpbin): - s = requests.Session() - r1 = s.get(httpbin('redirect/2'), allow_redirects=False, stream=True) + req = requests.Request('GET', httpbin('redirect/2')).prepare() + r1 = s.send(req, allow_redirects=False, stream=True) assert r1.is_redirect - rg = s.resolve_redirects(r1, r1.request, stream=True) - + rg = s.resolve_redirects(r1, req, stream=True) # read only the first eight bytes of the response body, # then follow the redirect r1.iter_content(8) r2 = next(rg) assert r2.is_redirect - # read all of the response via iter_content, # then follow the redirect for _ in r2.iter_content(): @@ -1642,40 +1766,47 @@ class TestRequests: r3 = next(rg) assert not r3.is_redirect - def test_prepare_body_position_non_stream(self): + def test_prepare_body_position_non_stream(self, s): data = b'the data' - prep = requests.Request('GET', 'http://example.com', data=data).prepare() + prep = requests.Request( + 'GET', 'http://example.com', data=data + ).prepare( + ) assert prep._body_position is None - def test_rewind_body(self): + def test_rewind_body(self, s): data = io.BytesIO(b'the data') - prep = requests.Request('GET', 'http://example.com', data=data).prepare() + prep = requests.Request( + 'GET', 'http://example.com', data=data + ).prepare( + ) assert prep._body_position == 0 assert prep.body.read() == b'the data' - # the data has all been read assert prep.body.read() == b'' - # rewind it back requests.utils.rewind_body(prep) assert prep.body.read() == b'the data' - def test_rewind_partially_read_body(self): + def test_rewind_partially_read_body(self, s): data = io.BytesIO(b'the data') data.read(4) # read some data - prep = requests.Request('GET', 'http://example.com', data=data).prepare() + prep = requests.Request( + 'GET', 'http://example.com', data=data + ).prepare( + ) assert prep._body_position == 4 assert prep.body.read() == b'data' - # the data has all been read assert prep.body.read() == b'' - # rewind it back requests.utils.rewind_body(prep) assert prep.body.read() == b'data' - def test_rewind_body_no_seek(self): + def test_rewind_body_no_seek(self, s): + class BadFileObj: + def __init__(self, data): self.data = data @@ -1686,16 +1817,20 @@ class TestRequests: return data = BadFileObj('the data') - prep = requests.Request('GET', 'http://example.com', data=data).prepare() - assert prep._body_position == 0 + prep = requests.Request( + 'GET', 'http://example.com', data=data + ).prepare( + ) + assert prep._body_position == 0 with pytest.raises(UnrewindableBodyError) as e: requests.utils.rewind_body(prep) - assert 'Unable to rewind request body' in str(e) - def test_rewind_body_failed_seek(self): + def test_rewind_body_failed_seek(self, s): + class BadFileObj: + def __init__(self, data): self.data = data @@ -1709,16 +1844,19 @@ class TestRequests: return data = BadFileObj('the data') - prep = requests.Request('GET', 'http://example.com', data=data).prepare() + prep = requests.Request( + 'GET', 'http://example.com', data=data + ).prepare( + ) assert prep._body_position == 0 - with pytest.raises(UnrewindableBodyError) as e: requests.utils.rewind_body(prep) - assert 'error occurred when rewinding request body' in str(e) - def test_rewind_body_failed_tell(self): + def test_rewind_body_failed_tell(self, s): + class BadFileObj: + def __init__(self, data): self.data = data @@ -1729,12 +1867,13 @@ class TestRequests: return data = BadFileObj('the data') - prep = requests.Request('GET', 'http://example.com', data=data).prepare() + prep = requests.Request( + 'GET', 'http://example.com', data=data + ).prepare( + ) assert prep._body_position is not None - with pytest.raises(UnrewindableBodyError) as e: requests.utils.rewind_body(prep) - assert 'Unable to rewind request body' in str(e) def _patch_adapter_gzipped_redirect(self, session, url): @@ -1751,18 +1890,25 @@ class TestRequests: adapter.build_response = build_response - def test_redirect_with_wrong_gzipped_header(self, httpbin): - s = requests.Session() + def test_redirect_with_wrong_gzipped_header(self, httpbin, s): url = httpbin('redirect/1') self._patch_adapter_gzipped_redirect(s, url) s.get(url) @pytest.mark.parametrize( - 'username, password, auth_str', ( + 'username, password, auth_str', + ( ('test', 'test', 'Basic dGVzdDp0ZXN0'), - (u'имя'.encode('utf-8'), u'пароль'.encode('utf-8'), 'Basic 0LjQvNGPOtC/0LDRgNC+0LvRjA=='), - )) - def test_basic_auth_str_is_always_native(self, username, password, auth_str): + ( + u'имя'.encode('utf-8'), + u'пароль'.encode('utf-8'), + 'Basic 0LjQvNGPOtC/0LDRgNC+0LvRjA==', + ), + ), + ) + def test_basic_auth_str_is_always_native( + self, username, password, auth_str + ): s = _basic_auth_str(username, password) assert isinstance(s, builtin_str) assert s == auth_str @@ -1776,41 +1922,40 @@ class TestRequests: i += 1 def test_json_param_post_content_type_works(self, httpbin): - r = requests.post( - httpbin('post'), - json={'life': 42} - ) + r = requests.post(httpbin('post'), json={'life': 42}) assert r.status_code == 200 assert 'application/json' in r.request.headers['Content-Type'] assert {'life': 42} == r.json()['json'] def test_json_param_post_should_not_override_data_param(self, httpbin): - r = requests.Request(method='POST', url=httpbin('post'), - data={'stuff': 'elixr'}, - json={'music': 'flute'}) + r = requests.Request( + method='POST', + url=httpbin('post'), + data={'stuff': 'elixr'}, + json={'music': 'flute'}, + ) prep = r.prepare() assert 'stuff=elixr' == prep.body - def test_response_iter_lines(self, httpbin): + @pytest.mark.parametrize('decode_unicode', (True, False)) + def test_response_iter_lines(self, httpbin, decode_unicode): r = requests.get(httpbin('stream/4'), stream=True) assert r.status_code == 200 - - it = r.iter_lines() + r.encoding = 'utf-8' + it = r.iter_lines(decode_unicode=decode_unicode) next(it) assert len(list(it)) == 3 def test_response_context_manager(self, httpbin): with requests.get(httpbin('stream/4'), stream=True) as response: assert isinstance(response, requests.Response) - assert response.raw.closed - def test_unconsumed_session_response_closes_connection(self, httpbin): - s = requests.session() - - with contextlib.closing(s.get(httpbin('stream/4'), stream=True)) as response: + def test_unconsumed_session_response_closes_connection(self, httpbin, s): + with contextlib.closing( + s.get(httpbin('stream/4'), stream=True) + ) as response: pass - assert response._content_consumed is False assert response.raw.closed @@ -1819,18 +1964,58 @@ class TestRequests: """Response.iter_lines() is not reentrant safe""" r = requests.get(httpbin('stream/4'), stream=True) assert r.status_code == 200 - next(r.iter_lines()) assert len(list(r.iter_lines())) == 3 - def test_session_close_proxy_clear(self, mocker): - proxies = { - 'one': mocker.Mock(), - 'two': mocker.Mock(), - } - session = requests.Session() - mocker.patch.dict(session.adapters['http://'].proxy_manager, proxies) - session.close() + def test_environment_comes_after_session(self, httpbin, s): + """The Session arguments should come before environment arguments.""" + # We get proxies from the environment and verify from the argument. + a = SendRecordingAdapter() + s.mount('http://', a) + # Both of these arguments are safe fallbacks that we can easily + # detect, but which will allow the request to succeed. + s.verify = False + s.proxies = {'http': None} + old_proxy = os.environ.get('HTTP_PROXY') + old_bundle = os.environ.get('REQUESTS_CA_BUNDLE') + try: + os.environ['HTTP_PROXY'] = '10.10.10.10:3128' + os.environ['REQUESTS_CA_BUNDLE'] = '/path/to/nowhere' + s.get(httpbin('get'), timeout=5) + finally: + if old_proxy is not None: + os.environ['HTTP_PROXY'] = old_proxy + else: + del os.environ['HTTP_PROXY'] + if old_bundle is not None: + os.environ['REQUESTS_CA_BUNDLE'] = old_bundle + else: + del os.environ['REQUESTS_CA_BUNDLE'] + call = a.send_calls[0] + assert call[1]['verify'] == False + proxies = call[1]['proxies'] + with pytest.raises(KeyError): + proxies['http'] + + @pytest.fixture(autouse=True) + def test_merge_environment_settings_verify(self, monkeypatch, s): + """Assert CA environment settings are merged as expected when missing""" + monkeypatch.delenv('CURL_CA_BUNDLE', raising=False) + monkeypatch.delenv('REQUESTS_CA_BUNDLE', raising=False) + assert s.trust_env is True + assert s.verify is True + assert 'REQUESTS_CA_BUNDLE' not in os.environ + assert 'CURL_CA_BUNDLE' not in os.environ + merged_settings = s.merge_environment_settings( + 'http://example.com', {}, False, True, None + ) + assert merged_settings['verify'] is True + + def test_session_close_proxy_clear(self, mocker, s): + proxies = {'one': mocker.Mock(), 'two': mocker.Mock()} + + mocker.patch.dict(s.adapters['http://'].proxy_manager, proxies) + s.close() proxies['one'].clear.assert_called_once_with() proxies['two'].clear.assert_called_once_with() @@ -1850,7 +2035,6 @@ class TestRequests: r.status_code = 0 r._content = False r._content_consumed = False - assert r.content is None with pytest.raises(ValueError): r.json() @@ -1865,7 +2049,35 @@ class TestRequests: resp.close() assert resp.raw.closed - def test_empty_stream_with_auth_does_not_set_content_length_header(self, httpbin): + def test_updating_ca_cert(self, httpbin_secure, s): + """Assert that requests use the latest configured CA certificates.""" + s.verify = pytest_httpbin.certs.where() + s.get(httpbin_secure('/')) + s.verify = True + with pytest.raises(requests.exceptions.SSLError) as e: + s.get(httpbin_secure('/')) + assert 'certificate verify failed' in str(e) + + def test_updating_client_cert(self, httpbin_secure, s): + """Assert that requests use the latest configured client certificates.""" + ca_file = pytest_httpbin.certs.where() + cert_dir = os.path.dirname(ca_file) + # All we need is a valid certificate and key to make a request. httpbin_secure + # won't check the signature or subject name, so it's okay that these happen to + # be the server's certificate and key. + cert = os.path.join(cert_dir, 'cert.pem') + key = os.path.join(cert_dir, 'key.pem') + + s.verify = ca_file + resp = s.get(httpbin_secure('/')) + resp_with_cert = s.get(httpbin_secure('/'), cert=(cert, key)) + assert resp_with_cert.raw._pool.cert_file == cert + assert resp_with_cert.raw._pool.key_file == key + assert resp.raw._pool is not resp_with_cert.raw._pool + + def test_empty_stream_with_auth_does_not_set_content_length_header( + self, httpbin + ): """Ensure that a byte stream with size 0 will not set both a Content-Length and Transfer-Encoding header. """ @@ -1877,7 +2089,9 @@ class TestRequests: assert 'Transfer-Encoding' in prepared_request.headers assert 'Content-Length' not in prepared_request.headers - def test_stream_with_auth_does_not_set_transfer_encoding_header(self, httpbin): + def test_stream_with_auth_does_not_set_transfer_encoding_header( + self, httpbin + ): """Ensure that a byte stream with size > 0 will not set both a Content-Length and Transfer-Encoding header. """ @@ -1900,6 +2114,69 @@ class TestRequests: assert 'Transfer-Encoding' in prepared_request.headers assert 'Content-Length' not in prepared_request.headers + def test_chunked_upload_with_manually_set_content_length_header_raises_error( + self, httpbin + ): + """Ensure that if a user manually sets a content length header, when + the data is chunked, that an InvalidHeader error is raised. + """ + data = (i for i in [b'a', b'b', b'c']) + url = httpbin('post') + with pytest.raises(InvalidHeader): + r = requests.post( + url, data=data, headers={'Content-Length': 'foo'} + ) + + def test_content_length_with_manually_set_transfer_encoding_raises_error( + self, httpbin + ): + """Ensure that if a user manually sets a Transfer-Encoding header when + data is not chunked that an InvalidHeader error is raised. + """ + data = 'test data' + url = httpbin('post') + with pytest.raises(InvalidHeader): + r = requests.post( + url, data=data, headers={'Transfer-Encoding': 'chunked'} + ) + + def test_null_body_does_not_raise_error(self, httpbin): + url = httpbin('post') + try: + requests.post(url, data=None) + except InvalidHeader: + pytest.fail('InvalidHeader error raised unexpectedly.') + + @pytest.mark.parametrize( + 'body, expected', + ( + (None, ('Content-Length', '0')), + ('test_data', ('Content-Length', '9')), + (io.BytesIO(b'test_data'), ('Content-Length', '9')), + (StringIO.StringIO(''), ('Transfer-Encoding', 'chunked')), + ), + ) + def test_prepare_content_length(self, httpbin, body, expected): + """Test prepare_content_length creates expected header.""" + prep = requests.PreparedRequest() + prep.headers = {} + prep.method = 'POST' + # Ensure Content-Length is set appropriately. + key, value = expected + prep.prepare_content_length(body) + assert prep.headers[key] == value + + def test_prepare_content_length_with_bad_body(self, httpbin): + """Test prepare_content_length raises exception with unsendable body.""" + # Initialize minimum required PreparedRequest. + prep = requests.PreparedRequest() + prep.headers = {} + prep.method = 'POST' + with pytest.raises(InvalidBodyError) as e: + # Send object that isn't iterable and has no accessible content. + prep.prepare_content_length(object()) + assert "Non-null body must have length or be streamable." in str(e) + def test_custom_redirect_mixin(self, httpbin): """Tests a custom mixin to overwrite ``get_redirect_target``. @@ -1914,23 +2191,25 @@ class TestRequests: """ url_final = httpbin('html') querystring_malformed = urlencode({'location': url_final}) - url_redirect_malformed = httpbin('response-headers?%s' % querystring_malformed) + url_redirect_malformed = httpbin( + 'response-headers?%s' % querystring_malformed + ) querystring_redirect = urlencode({'url': url_redirect_malformed}) url_redirect = httpbin('redirect-to?%s' % querystring_redirect) - urls_test = [url_redirect, - url_redirect_malformed, - url_final, - ] + urls_test = [url_redirect, url_redirect_malformed, url_final] class CustomRedirectSession(requests.Session): + def get_redirect_target(self, resp): # default behavior if resp.is_redirect: return resp.headers['location'] + # edge case - check to see if 'location' is in headers anyways location = resp.headers.get('location') if location and (location != resp.url): return location + return None session = CustomRedirectSession() @@ -1943,15 +2222,44 @@ class TestRequests: assert not r.history[1].is_redirect assert r.url == urls_test[2] + def test_multiple_response_headers_with_same_name_same_case(self, httpbin): + qs = 'Fruit=Apple&Fruit=Blood+Orange&Fruit=Banana&Fruit=Berry,+Blue' + resp = requests.get(httpbin('response-headers?' + qs)) + fruits = resp.headers['fruit'] + assert fruits == 'Apple, Blood Orange, Banana, Berry, Blue' + # As we are using HTTPHeaderDict, we should be able to extract the + # individual header values too. + assert resp.headers.getlist('fruit') == [ + 'Apple', 'Blood Orange', 'Banana', 'Berry, Blue' + ] + + def test_multiple_response_headers_with_same_name_diff_case(self, httpbin): + # urllib3 seems to have trouble guaranteeing the order of the items when + # the case is different, so we just need to make sure all of the items + # are there, rather than asserting a particular order. + qs = 'Fruit=Apple&Fruit=Blood+Orange&Fruit=Banana&Fruit=Berry,+Blue' + resp = requests.get(httpbin('response-headers?' + qs)) + # These are all possible acceptable combinations for the header. + fruit_choices = ['Apple', 'Blood Orange', 'Banana', 'Berry, Blue'] + fruit_permutations = itertools.permutations(fruit_choices) + fruit_multiheaders = [list(fp) for fp in fruit_permutations] + fruit_headers = set(', '.join(fp) for fp in fruit_multiheaders) + assert resp.headers['fruit'] in fruit_headers + # As we are using HTTPHeaderDict, we should be able to extract the + # individual header values too. + assert resp.headers.getlist('fruit') in fruit_multiheaders + class TestCaseInsensitiveDict: @pytest.mark.parametrize( - 'cid', ( + 'cid', + ( CaseInsensitiveDict({'Foo': 'foo', 'BAr': 'bar'}), CaseInsensitiveDict([('Foo', 'foo'), ('BAr', 'bar')]), CaseInsensitiveDict(FOO='foo', BAr='bar'), - )) + ), + ) def test_init(self, cid): assert len(cid) == 2 assert 'foo' in cid @@ -2045,29 +2353,26 @@ class TestCaseInsensitiveDict: assert cid.setdefault('notspam', 'notblueval') == 'notblueval' def test_lower_items(self): - cid = CaseInsensitiveDict({ - 'Accept': 'application/json', - 'user-Agent': 'requests', - }) + cid = CaseInsensitiveDict( + {'Accept': 'application/json', 'user-Agent': 'requests'} + ) keyset = frozenset(lowerkey for lowerkey, v in cid.lower_items()) lowerkeyset = frozenset(['accept', 'user-agent']) assert keyset == lowerkeyset def test_preserve_key_case(self): - cid = CaseInsensitiveDict({ - 'Accept': 'application/json', - 'user-Agent': 'requests', - }) + cid = CaseInsensitiveDict( + {'Accept': 'application/json', 'user-Agent': 'requests'} + ) keyset = frozenset(['Accept', 'user-Agent']) assert frozenset(i[0] for i in cid.items()) == keyset assert frozenset(cid.keys()) == keyset assert frozenset(cid) == keyset def test_preserve_last_key_case(self): - cid = CaseInsensitiveDict({ - 'Accept': 'application/json', - 'user-Agent': 'requests', - }) + cid = CaseInsensitiveDict( + {'Accept': 'application/json', 'user-Agent': 'requests'} + ) cid.update({'ACCEPT': 'application/json'}) cid['USER-AGENT'] = 'requests' keyset = frozenset(['ACCEPT', 'USER-AGENT']) @@ -2076,32 +2381,40 @@ class TestCaseInsensitiveDict: assert frozenset(cid) == keyset def test_copy(self): - cid = CaseInsensitiveDict({ - 'Accept': 'application/json', - 'user-Agent': 'requests', - }) + cid = CaseInsensitiveDict( + {'Accept': 'application/json', 'user-Agent': 'requests'} + ) cid_copy = cid.copy() assert cid == cid_copy cid['changed'] = True assert cid != cid_copy + def test_url_surrounding_whitespace(self, httpbin): + """Test case with URLs surrounded by whitespace characters.""" + get_url = httpbin('get') + # All surrounding whitespaces are supposed to be ignored: + assert requests.get(get_url + ' ').status_code == 200 + assert requests.get(' ' + get_url).status_code == 200 + assert requests.get(get_url + ' \t ').status_code == 200 + assert requests.get(' \t' + get_url).status_code == 200 + assert requests.get(get_url + '\n').status_code == 200 + # The whitespaces can't be in the middle of the URL though: + assert requests.get(get_url + ' abc').status_code == 404 + class TestMorselToCookieExpires: """Tests for morsel_to_cookie when morsel contains expires.""" def test_expires_valid_str(self): """Test case where we convert expires from string time.""" - morsel = Morsel() morsel['expires'] = 'Thu, 01-Jan-1970 00:00:01 GMT' cookie = morsel_to_cookie(morsel) assert cookie.expires == 1 @pytest.mark.parametrize( - 'value, exception', ( - (100, TypeError), - ('woops', ValueError), - )) + 'value, exception', ((100, TypeError), ('woops', ValueError)) + ) def test_expires_invalid_int(self, value, exception): """Test case where an invalid type is passed for expires.""" morsel = Morsel() @@ -2111,7 +2424,6 @@ class TestMorselToCookieExpires: def test_expires_none(self): """Test case where expires is None.""" - morsel = Morsel() morsel['expires'] = None cookie = morsel_to_cookie(morsel) @@ -2119,12 +2431,10 @@ class TestMorselToCookieExpires: class TestMorselToCookieMaxAge: - """Tests for morsel_to_cookie when morsel contains max-age.""" def test_max_age_valid_int(self): """Test case where a valid max age in seconds is passed.""" - morsel = Morsel() morsel['max-age'] = 60 cookie = morsel_to_cookie(morsel) @@ -2132,7 +2442,6 @@ class TestMorselToCookieMaxAge: def test_max_age_invalid_str(self): """Test case where a invalid max age is passed.""" - morsel = Morsel() morsel['max-age'] = 'woops' with pytest.raises(TypeError): @@ -2148,20 +2457,20 @@ class TestTimeout: assert 'Read timed out' in e.args[0].args[0] @pytest.mark.parametrize( - 'timeout, error_text', ( + 'timeout, error_text', + ( ((3, 4, 5), '(connect, read)'), ('foo', 'must be an int, float or None'), - )) + ), + ) def test_invalid_timeout(self, httpbin, timeout, error_text): with pytest.raises(ValueError) as e: requests.get(httpbin('get'), timeout=timeout) assert error_text in str(e) @pytest.mark.parametrize( - 'timeout', ( - None, - Urllib3Timeout(connect=None, read=None) - )) + 'timeout', (None, Urllib3Timeout(connect=None, read=None)) + ) def test_none_timeout(self, httpbin, timeout): """Check that you can set None as a valid timeout value. @@ -2175,10 +2484,8 @@ class TestTimeout: assert r.status_code == 200 @pytest.mark.parametrize( - 'timeout', ( - (None, 0.1), - Urllib3Timeout(connect=None, read=0.1) - )) + 'timeout', ((None, 0.1), Urllib3Timeout(connect=None, read=0.1)) + ) def test_read_timeout(self, httpbin, timeout): try: requests.get(httpbin('delay/10'), timeout=timeout) @@ -2187,10 +2494,8 @@ class TestTimeout: pass @pytest.mark.parametrize( - 'timeout', ( - (0.1, None), - Urllib3Timeout(connect=0.1, read=None) - )) + 'timeout', ((0.1, None), Urllib3Timeout(connect=0.1, read=None)) + ) def test_connect_timeout(self, timeout): try: requests.get(TARPIT, timeout=timeout) @@ -2200,10 +2505,8 @@ class TestTimeout: assert isinstance(e, Timeout) @pytest.mark.parametrize( - 'timeout', ( - (0.1, 0.1), - Urllib3Timeout(connect=0.1, read=0.1) - )) + 'timeout', ((0.1, 0.1), Urllib3Timeout(connect=0.1, read=0.1)) + ) def test_total_timeout_connect(self, timeout): try: requests.get(TARPIT, timeout=timeout) @@ -2221,12 +2524,14 @@ SendCall = collections.namedtuple('SendCall', ('args', 'kwargs')) class RedirectSession(SessionRedirectMixin): + def __init__(self, order_of_redirects): self.redirects = order_of_redirects self.calls = [] self.max_redirects = 30 self.cookies = {} self.trust_env = False + self.location = '/' def send(self, *args, **kwargs): self.calls.append(SendCall(args, kwargs)) @@ -2235,13 +2540,11 @@ class RedirectSession(SessionRedirectMixin): def build_response(self): request = self.calls[-1].args[0] r = requests.Response() - try: r.status_code = int(self.redirects.pop(0)) except IndexError: r.status_code = 200 - - r.headers = CaseInsensitiveDict({'Location': '/'}) + r.headers = CaseInsensitiveDict({'Location': self.location}) r.raw = self._build_raw() r.request = request return r @@ -2256,11 +2559,7 @@ def test_json_encodes_as_bytes(): # urllib3 expects bodies as bytes-like objects body = {"key": "value"} p = PreparedRequest() - p.prepare( - method='GET', - url='https://www.example.com/', - json=body - ) + p.prepare(method='GET', url='https://www.example.com/', json=body) assert isinstance(p.body, bytes) @@ -2285,33 +2584,35 @@ def test_requests_are_updated_each_time(httpbin): assert session.calls[-1] == send_call -@pytest.mark.parametrize("var,url,proxy", [ - ('http_proxy', 'http://example.com', 'socks5://proxy.com:9876'), - ('https_proxy', 'https://example.com', 'socks5://proxy.com:9876'), - ('all_proxy', 'http://example.com', 'socks5://proxy.com:9876'), - ('all_proxy', 'https://example.com', 'socks5://proxy.com:9876'), -]) -def test_proxy_env_vars_override_default(var, url, proxy): - session = requests.Session() +@pytest.mark.parametrize( + "var,url,proxy", + [ + ('http_proxy', 'http://example.com', 'socks5://proxy.com:9876'), + ('https_proxy', 'https://example.com', 'socks5://proxy.com:9876'), + ('all_proxy', 'http://example.com', 'socks5://proxy.com:9876'), + ('all_proxy', 'https://example.com', 'socks5://proxy.com:9876'), + ], +) +def test_proxy_env_vars_override_default(var, url, proxy, s): + prep = PreparedRequest() prep.prepare(method='GET', url=url) - - kwargs = { - var: proxy - } + kwargs = {var: proxy} scheme = urlparse(url).scheme with override_environ(**kwargs): - proxies = session.rebuild_proxies(prep, {}) + proxies = s.rebuild_proxies(prep, {}) assert scheme in proxies assert proxies[scheme] == proxy @pytest.mark.parametrize( - 'data', ( + 'data', + ( (('a', 'b'), ('c', 'd')), (('c', 'd'), ('a', 'b')), (('a', 'b'), ('c', 'd'), ('e', 'f')), - )) + ), +) def test_data_argument_accepts_tuples(data): """Ensure that the data argument will accept tuples of strings and properly encode them. @@ -2321,32 +2622,31 @@ def test_data_argument_accepts_tuples(data): method='GET', url='http://www.example.com', data=data, - hooks=default_hooks() + hooks=default_hooks(), ) assert p.body == urlencode(data) @pytest.mark.parametrize( - 'kwargs', ( + 'kwargs', + ( None, { 'method': 'GET', 'url': 'http://www.example.com', 'data': 'foo=bar', - 'hooks': default_hooks() + 'hooks': default_hooks(), }, { 'method': 'GET', 'url': 'http://www.example.com', 'data': 'foo=bar', 'hooks': default_hooks(), - 'cookies': {'foo': 'bar'} + 'cookies': {'foo': 'bar'}, }, - { - 'method': 'GET', - 'url': u('http://www.example.com/üniçø∂é') - }, - )) + {'method': 'GET', 'url': u('http://www.example.com/üniçø∂é')}, + ), +) def test_prepared_copy(kwargs): p = PreparedRequest() if kwargs: @@ -2356,21 +2656,28 @@ def test_prepared_copy(kwargs): assert getattr(p, attr) == getattr(copy, attr) -def test_urllib3_retries(httpbin): - from urllib3.util import Retry - s = requests.Session() - s.mount('http://', HTTPAdapter(max_retries=Retry( - total=2, status_forcelist=[500] - ))) +def test_prepare_requires_a_request_method(): + req = requests.Request() + with pytest.raises(ValueError): + req.prepare() + prepped = PreparedRequest() + with pytest.raises(ValueError): + prepped.prepare() + +def test_urllib3_retries(httpbin, s): + from urllib3.util import Retry + + s.mount( + 'http://', + HTTPAdapter(max_retries=Retry(total=2, status_forcelist=[500])), + ) with pytest.raises(RetryError): s.get(httpbin('status/500')) -def test_urllib3_pool_connection_closed(httpbin): - s = requests.Session() +def test_urllib3_pool_connection_closed(httpbin, s): s.mount('http://', HTTPAdapter(pool_connections=0, pool_maxsize=0)) - try: s.get(httpbin('status/200')) except ConnectionError as e: @@ -2378,6 +2685,7 @@ def test_urllib3_pool_connection_closed(httpbin): class TestPreparingURLs(object): + @pytest.mark.parametrize( 'url,expected', ( @@ -2386,47 +2694,44 @@ class TestPreparingURLs(object): (u'http://xn--n3h.net/', u'http://xn--n3h.net/'), ( u'http://ジェーピーニック.jp'.encode('utf-8'), - u'http://xn--hckqz9bzb1cyrb.jp/' + u'http://xn--hckqz9bzb1cyrb.jp/', ), ( u'http://straße.de/straße', - u'http://xn--strae-oqa.de/stra%C3%9Fe' + u'http://xn--strae-oqa.de/stra%C3%9Fe', ), ( u'http://straße.de/straße'.encode('utf-8'), - u'http://xn--strae-oqa.de/stra%C3%9Fe' + u'http://xn--strae-oqa.de/stra%C3%9Fe', ), ( u'http://Königsgäßchen.de/straße', - u'http://xn--knigsgchen-b4a3dun.de/stra%C3%9Fe' + u'http://xn--knigsgchen-b4a3dun.de/stra%C3%9Fe', ), ( u'http://Königsgäßchen.de/straße'.encode('utf-8'), - u'http://xn--knigsgchen-b4a3dun.de/stra%C3%9Fe' - ), - ( - b'http://xn--n3h.net/', - u'http://xn--n3h.net/' + u'http://xn--knigsgchen-b4a3dun.de/stra%C3%9Fe', ), + (b'http://xn--n3h.net/', u'http://xn--n3h.net/'), ( b'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/', - u'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/' + u'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/', ), ( u'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/', - u'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/' - ) - ) + u'http://[1200:0000:ab00:1234:0000:2552:7777:1313]:12345/', + ), + ), ) def test_preparing_url(self, url, expected): def normalize_percent_encode(x): - # Helper function that normalizes equivalent + # Helper function that normalizes equivalent # percent-encoded bytes before comparisons for c in re.findall(r'%[a-fA-F0-9]{2}', x): x = x.replace(c, c.upper()) return x - + r = requests.Request('GET', url=url) p = r.prepare() assert normalize_percent_encode(p.url) == expected @@ -2438,8 +2743,8 @@ class TestPreparingURLs(object): b"http://*", u"http://*.google.com", u"http://*", - u"http://☃.net/" - ) + u"http://☃.net/", + ), ) def test_preparing_bad_url(self, url): r = requests.Request('GET', url=url) @@ -2458,28 +2763,16 @@ class TestPreparingURLs(object): @pytest.mark.parametrize( 'input, expected', - ( - ( - b"http+unix://%2Fvar%2Frun%2Fsocket/path%7E", - u"http+unix://%2Fvar%2Frun%2Fsocket/path~", - ), - ( - u"http+unix://%2Fvar%2Frun%2Fsocket/path%7E", - u"http+unix://%2Fvar%2Frun%2Fsocket/path~", - ), - ( - b"mailto:user@example.org", - u"mailto:user@example.org", - ), - ( - u"mailto:user@example.org", - u"mailto:user@example.org", - ), - ( - b"data:SSDimaUgUHl0aG9uIQ==", - u"data:SSDimaUgUHl0aG9uIQ==", - ) - ) + ((b"mailto:user@example.org", u"mailto:user@example.org"), (u"mailto:user@example.org", u"mailto:user@example.org"), (b"data:SSDimaUgUHl0aG9uIQ==", u"data:SSDimaUgUHl0aG9uIQ==")), + # TODO: Bugs in rfc3986, apparently. + # ( + # b"http+unix://%2Fvar%2Frun%2Fsocket/path%7E", + # u"http+unix://%2Fvar%2Frun%2Fsocket/path~", + # ), + # ( + # u"http+unix://%2Fvar%2Frun%2Fsocket/path%7E", + # u"http+unix://%2Fvar%2Frun%2Fsocket/path~", + # ), ) def test_url_mutation(self, input, expected): """ @@ -2494,28 +2787,18 @@ class TestPreparingURLs(object): @pytest.mark.parametrize( 'input, params, expected', - ( - ( - b"http+unix://%2Fvar%2Frun%2Fsocket/path", - {"key": "value"}, - u"http+unix://%2Fvar%2Frun%2Fsocket/path?key=value", - ), - ( - u"http+unix://%2Fvar%2Frun%2Fsocket/path", - {"key": "value"}, - u"http+unix://%2Fvar%2Frun%2Fsocket/path?key=value", - ), - ( - b"mailto:user@example.org", - {"key": "value"}, - u"mailto:user@example.org", - ), - ( - u"mailto:user@example.org", - {"key": "value"}, - u"mailto:user@example.org", - ), - ) + ((b"mailto:user@example.org", {"key": "value"}, u"mailto:user@example.org"), (u"mailto:user@example.org", {"key": "value"}, u"mailto:user@example.org")), + # TODO: + # ( + # b"http+unix://%2Fvar%2Frun%2Fsocket/path", + # {"key": "value"}, + # u"http+unix://%2Fvar%2Frun%2Fsocket/path?key=value", + # ), + # ( + # u"http+unix://%2Fvar%2Frun%2Fsocket/path", + # {"key": "value"}, + # u"http+unix://%2Fvar%2Frun%2Fsocket/path?key=value", + # ), ) def test_parameters_for_nonstandard_schemes(self, input, params, expected): """ @@ -2525,3 +2808,219 @@ class TestPreparingURLs(object): r = requests.Request('GET', url=input, params=params) p = r.prepare() assert p.url == expected + + +class TestGetConnection(object): + """ + Tests for the :meth:`requests.adapters.HTTPAdapter.get_connection` that assert + the connections are correctly configured. + """ + + @pytest.mark.parametrize( + 'proxies, verify, cert, expected', + ( + ( + {}, + True, + None, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': DEFAULT_CA_BUNDLE_PATH, + 'ca_cert_dir': None, + 'cert_file': None, + 'key_file': None, + }, + ), + ( + {}, + False, + None, + { + 'cert_reqs': 'CERT_NONE', + 'ca_certs': None, + 'ca_cert_dir': None, + 'cert_file': None, + 'key_file': None, + }, + ), + ( + {}, + __file__, + None, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': __file__, + 'ca_cert_dir': None, + 'cert_file': None, + 'key_file': None, + }, + ), + ( + {}, + os.path.dirname(__file__), + None, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': None, + 'ca_cert_dir': os.path.dirname(__file__), + 'cert_file': None, + 'key_file': None, + }, + ), + ( + {}, + True, + None, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': DEFAULT_CA_BUNDLE_PATH, + 'ca_cert_dir': None, + 'cert_file': None, + 'key_file': None, + }, + ), + ( + {}, + True, + __file__, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': DEFAULT_CA_BUNDLE_PATH, + 'ca_cert_dir': None, + 'cert_file': __file__, + 'key_file': None, + }, + ), + ( + {}, + True, + (__file__, __file__), + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': DEFAULT_CA_BUNDLE_PATH, + 'ca_cert_dir': None, + 'cert_file': __file__, + 'key_file': __file__, + }, + ), + ( + {}, + True, + (__file__, __file__), + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': DEFAULT_CA_BUNDLE_PATH, + 'ca_cert_dir': None, + 'cert_file': __file__, + 'key_file': __file__, + }, + ), + ( + { + 'http': 'http://proxy.example.com', + 'https': 'http://proxy.example.com', + }, + True, + None, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': DEFAULT_CA_BUNDLE_PATH, + 'ca_cert_dir': None, + 'cert_file': None, + 'key_file': None, + }, + ), + ( + { + 'http': 'http://proxy.example.com', + 'https': 'http://proxy.example.com', + }, + os.path.dirname(__file__), + None, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': None, + 'ca_cert_dir': os.path.dirname(__file__), + 'cert_file': None, + 'key_file': None, + }, + ), + ( + { + 'http': 'http://proxy.example.com', + 'https': 'http://proxy.example.com', + }, + __file__, + None, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': __file__, + 'ca_cert_dir': None, + 'cert_file': None, + 'key_file': None, + }, + ), + ( + { + 'http': 'http://proxy.example.com', + 'https': 'http://proxy.example.com', + }, + True, + __file__, + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': DEFAULT_CA_BUNDLE_PATH, + 'ca_cert_dir': None, + 'cert_file': __file__, + 'key_file': None, + }, + ), + ( + { + 'http': 'http://proxy.example.com', + 'https': 'http://proxy.example.com', + }, + True, + (__file__, __file__), + { + 'cert_reqs': 'CERT_REQUIRED', + 'ca_certs': DEFAULT_CA_BUNDLE_PATH, + 'ca_cert_dir': None, + 'cert_file': __file__, + 'key_file': __file__, + }, + ), + ), + ) + def test_get_https_connection(self, proxies, verify, cert, expected): + """Assert connections are configured correctly.""" + adapter = requests.adapters.HTTPAdapter() + connection = adapter.get_connection( + 'https://example.com', proxies=proxies, verify=verify, cert=cert + ) + actual_config = {} + for key, value in connection.__dict__.items(): + if key in expected: + actual_config[key] = value + assert actual_config == expected + + @pytest.mark.parametrize( + 'verify, cert', + ( + ('a/path/that/does/not/exist', None), + (True, 'a/path/that/does/not/exist'), + (True, (__file__, 'a/path/that/does/not/exist')), + (True, ('a/path/that/does/not/exist', __file__)), + ), + ) + def test_cert_files_missing(self, verify, cert): + """ + Assert an IOError is raised when one of the certificate files or + directories can't be found. + """ + adapter = requests.adapters.HTTPAdapter() + with pytest.raises(IOError) as excinfo: + adapter.get_connection( + 'https://example.com', verify=verify, cert=cert + ) + excinfo.match('invalid path: a/path/that/does/not/exist') diff --git a/tests/test_structures.py b/tests/test_structures.py index e4d2459f..6d0dc356 100644 --- a/tests/test_structures.py +++ b/tests/test_structures.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- - import pytest -from requests.structures import CaseInsensitiveDict, LookupDict +from requests3.structures import CaseInsensitiveDict, LookupDict, HTTPHeaderDict +from urllib3._collections import HTTPHeaderDict as U3HeaderDict class TestCaseInsensitiveDict: @@ -16,7 +16,9 @@ class TestCaseInsensitiveDict: def test_list(self): assert list(self.case_insensitive_dict) == ['Accept'] - possible_keys = pytest.mark.parametrize('key', ('accept', 'ACCEPT', 'aCcEpT', 'Accept')) + possible_keys = pytest.mark.parametrize( + 'key', ('accept', 'ACCEPT', 'aCcEpT', 'Accept') + ) @possible_keys def test_getitem(self, key): @@ -28,10 +30,14 @@ class TestCaseInsensitiveDict: assert key not in self.case_insensitive_dict def test_lower_items(self): - assert list(self.case_insensitive_dict.lower_items()) == [('accept', 'application/json')] + assert list(self.case_insensitive_dict.lower_items()) == [ + ('accept', 'application/json') + ] def test_repr(self): - assert repr(self.case_insensitive_dict) == "{'Accept': 'application/json'}" + assert repr( + self.case_insensitive_dict + ) == "{'Accept': 'application/json'}" def test_copy(self): copy = self.case_insensitive_dict.copy() @@ -39,16 +45,186 @@ class TestCaseInsensitiveDict: assert copy == self.case_insensitive_dict @pytest.mark.parametrize( - 'other, result', ( - ({'AccePT': 'application/json'}, True), - ({}, False), - (None, False) - ) + 'other, result', + (({'AccePT': 'application/json'}, True), ({}, False), (None, False)), ) def test_instance_equality(self, other, result): assert (self.case_insensitive_dict == other) is result +class TestHTTPHeaderDictCompatibility(TestCaseInsensitiveDict): + """HTTPHeaderDict should be completely compatible with CaseInsensitiveDict + when used for headers, so ensure that all the tests for the base class + also pass for this one.""" + + @pytest.fixture(autouse=True) + def setup(self): + self.case_insensitive_dict = HTTPHeaderDict() + self.case_insensitive_dict['Accept'] = 'application/json' + + +class TestHTTPHeaderDict: + + @pytest.fixture(autouse=True) + def setup(self): + self.kvs = [ + ('animal', 'chicken'), + ('AnimaL', 'Cow'), + ('CAKE', 'Cheese!'), + ('Sauce', 'Bread'), + ('Sauce', 'Cherry, or Plum Tomato'), + ] + # HTTPHeaderDict from urllib3. + self.u3dict = ud = U3HeaderDict() + [ud.add(*tpl) for tpl in self.kvs] + # Regular dictionary. + self.ddict = dict(self.kvs) + self.ddict['Sauce'] = ['Bread!', 'Cherry, or Plum Tomato'] + # Used by test_extend. All of these "extra" values are mostly + # equivalent to each other. + self.extra_hd = hd2 = HTTPHeaderDict(ANIMAL=['Dog', 'elephant']) + hd2['cake'] = 'Babka' + hd2.setlist('sound', ['quiet', 'LOUD']) + hd2['CUTLERY'] = 'fork' + self.extra_tuple_pairs = tuple_pairs = [ + ('ANIMAL', 'Dog'), + ('Animal', 'elephant'), + ('cake', ['Babka']), + ('sound', 'quiet'), + ('sound', 'LOUD'), + ('CUTLERY', 'fork'), + ] + self.extra_simple_dict = dict(tuple_pairs) + self.extra_simple_dict['sound'] = ('quiet', 'LOUD') + self.extra_u3 = U3HeaderDict() + for k, v in tuple_pairs: + if isinstance(v, (tuple, list)): + for vi in v: + self.extra_u3.add(k, vi) + else: + self.extra_u3.add(k, v) + + def test_item_access(self): + hd = HTTPHeaderDict(self.kvs) + # Test that values are combined. + assert hd['Sauce'] == 'Bread, Cherry, or Plum Tomato' + assert hd['ANIMAL'] == 'chicken, Cow' + # Test we can overwrite values. + hd['animal'] = 'Goat!' + assert hd['anIMal'] == 'Goat!' + # Test deletion works. + del hd['sauce'] + pytest.raises(KeyError, hd.__getitem__, 'sauce') + # Only string types allowed. + pytest.raises(ValueError, hd.__setitem__, 'cake', ['Cheese', 'sponge']) + + def test_equality(self): + hd = HTTPHeaderDict(self.u3dict) + assert hd == self.u3dict + assert hd == HTTPHeaderDict(hd) + # Test that we still work even if we are comparing to a + # CaseInsensitiveDict instance. + cid = CaseInsensitiveDict(hd) + assert hd == cid + assert cid == hd + + def test_lower_items(self): + hd = HTTPHeaderDict(self.kvs, cutlery='fork') + assert list(hd.lower_items()) == [ + ('animal', 'chicken, Cow'), + ('cake', 'Cheese!'), + ('sauce', 'Bread, Cherry, or Plum Tomato'), + ('cutlery', 'fork'), + ] + + def test_copy(self): + hd = HTTPHeaderDict(self.u3dict) + hd2 = hd.copy() + assert hd is not hd2 + assert hd == hd2 + + def test_get_and_set_list(self): + hd = HTTPHeaderDict(self.kvs) + assert hd.getlist('SAUCE') == ['Bread', 'Cherry, or Plum Tomato'] + assert hd.getlist('CAKE') == ['Cheese!'] + assert hd.getlist('DRINK') == [] + # Needs to be a regular sequence type containing just strings. + pytest.raises(ValueError, hd.setlist, 'Drink', 'Water') + pytest.raises(ValueError, hd.setlist, 'Drink', ['H', 2, 'O']) + # Test multi-setting. + hd.setlist('Drink', ['Water', 'Juice']) + assert hd.getlist('DRINK') == ['Water', 'Juice'] + # Setting to an empty sequence should remove the entry. + hd.setlist('DRInk', []) + pytest.raises(KeyError, hd.__getitem__, 'DrinK') + assert hd.getlist('DRiNK') == [] + + def test_add(self): + hd = HTTPHeaderDict() + hd.add('sound', 'quiet') + hd.add('SOUND', 'LOUD') + assert hd.getlist('Sound') == ['quiet', 'LOUD'] + # Enforce type-checking in the add method. + pytest.raises(ValueError, hd.add, 'Sound', 5) + + @pytest.mark.parametrize( + 'attr,as_arg,animal_arg_is_ordered', + [('extra_hd', True, True), ('extra_tuple_pairs', True, True), ('extra_simple_dict', True, False), ('extra_u3', True, False), ('extra_simple_dict', False, False)], + # These types will have the "animal" arguments in our preferred order. + # And these types will lose the ordering, so we can't make assertions + # about the final order of those values. + ) + def test_extend(self, attr, as_arg, animal_arg_is_ordered): + item = getattr(self, attr) + # Call extend with the associated values - we should see all of the + # merged data in the HTTPHeaderDict instance. + extras = {'cutlery': 'knife'} + hd = HTTPHeaderDict(self.kvs) + if as_arg: + hd.extend(item, **extras) + else: + hd.extend(extras, **item) + # Test all the stored values are what we expect. + mget = hd.getlist + # Depending on the item we merged in, we might be able to make + # assumptions what the overall order of the structure is. + animal_seq = mget('animal') + if animal_arg_is_ordered: + assert animal_seq == ['chicken', 'Cow', 'Dog', 'elephant'] + else: + # The existing order in HTTPHeadersDict of the first two values + # should be preserved - no guarantees in which order the other + # two values are added. + assert animal_seq in [ + ['chicken', 'Cow', 'Dog', 'elephant'], + ['chicken', 'Cow', 'elephant', 'Dog'], + ] + assert mget('cake') == ['Cheese!', 'Babka'] + assert mget('sound') == ['quiet', 'LOUD'] + # We don't mandate the order in which these dictionaries are + # processed, so it's fine whichever order it is. + assert mget('cutlery') in [['fork', 'knife'], ['knife', 'fork']] + + def test_extend_type_checking(self): + hd = HTTPHeaderDict() + pytest.raises(ValueError, hd.extend, dict(type=['xml', None, 'html'])) + + def test_repr(self): + hd = HTTPHeaderDict() + assert repr(hd) == '{}' + hd.add('type', 'xml') + assert repr(hd) == "{'type': 'xml'}" + hd.add('type', 'html') + assert repr(hd) == "{'type': ('xml', 'html')}" + # We can't guarantee order once we have more than one key. + hd.add('Accept', 'text/html') + assert repr(hd) in [ + "{'type': ('xml', 'html'), 'Accept': 'text/html'}", + "{'Accept': 'text/html', 'type': ('xml', 'html')}", + ] + assert str(hd) == repr(hd) + + class TestLookupDict: @pytest.fixture(autouse=True) @@ -61,10 +237,7 @@ class TestLookupDict: assert repr(self.lookup_dict) == "" get_item_parameters = pytest.mark.parametrize( - 'key, value', ( - ('bad_gateway', 502), - ('not_a_key', None) - ) + 'key, value', (('bad_gateway', 502), ('not_a_key', None)) ) @get_item_parameters diff --git a/tests/test_testserver.py b/tests/test_testserver.py index aac52926..74cc0bad 100644 --- a/tests/test_testserver.py +++ b/tests/test_testserver.py @@ -1,11 +1,10 @@ # -*- coding: utf-8 -*- - import threading import socket import time import pytest -import requests +import requests3 as requests from tests.testserver.server import Server @@ -34,9 +33,7 @@ class TestTestServer: with Server.basic_response_server() as (host, port): sock = socket.socket() sock.connect((host, port)) - sock.close() - with pytest.raises(socket.error): new_sock = socket.socket() new_sock.connect((host, port)) @@ -44,14 +41,10 @@ class TestTestServer: def test_text_response(self): """the text_response_server sends the given text""" server = Server.text_response_server( - "HTTP/1.1 200 OK\r\n" + - "Content-Length: 6\r\n" + - "\r\nroflol" + "HTTP/1.1 200 OK\r\n" + "Content-Length: 6\r\n" + "\r\nroflol" ) - with server as (host, port): - r = requests.get('http://{}:{}'.format(host, port)) - + r = requests.get('http://{0}:{1}'.format(host, port)) assert r.status_code == 200 assert r.text == u'roflol' assert r.headers['Content-Length'] == '6' @@ -67,8 +60,11 @@ class TestTestServer: def test_basic_waiting_server(self): """the server waits for the block_server event to be set before closing""" block_server = threading.Event() - - with Server.basic_response_server(wait_to_close_event=block_server) as (host, port): + with Server.basic_response_server( + wait_to_close_event=block_server + ) as ( + host, port + ): sock = socket.socket() sock.connect((host, port)) sock.sendall(b'send something') @@ -79,67 +75,60 @@ class TestTestServer: def test_multiple_requests(self): """multiple requests can be served""" requests_to_handle = 5 - - server = Server.basic_response_server(requests_to_handle=requests_to_handle) - + server = Server.basic_response_server( + requests_to_handle=requests_to_handle + ) with server as (host, port): server_url = 'http://{}:{}'.format(host, port) for _ in range(requests_to_handle): r = requests.get(server_url) assert r.status_code == 200 - # the (n+1)th request fails with pytest.raises(requests.exceptions.ConnectionError): r = requests.get(server_url) - @pytest.mark.skip(reason="this fails non-deterministically under pytest-xdist") + @pytest.mark.skip( + reason="this fails non-deterministically under pytest-xdist" + ) def test_request_recovery(self): """can check the requests content""" # TODO: figure out why this sometimes fails when using pytest-xdist. server = Server.basic_response_server(requests_to_handle=2) first_request = b'put your hands up in the air' second_request = b'put your hand down in the floor' - with server as address: sock1 = socket.socket() sock2 = socket.socket() - sock1.connect(address) sock1.sendall(first_request) sock1.close() - sock2.connect(address) sock2.sendall(second_request) sock2.close() - assert server.handler_results[0] == first_request assert server.handler_results[1] == second_request def test_requests_after_timeout_are_not_received(self): """the basic response handler times out when receiving requests""" server = Server.basic_response_server(request_timeout=1) - with server as address: sock = socket.socket() sock.connect(address) time.sleep(1.5) sock.sendall(b'hehehe, not received') sock.close() - assert server.handler_results[0] == b'' def test_request_recovery_with_bigger_timeout(self): """a biggest timeout can be specified""" server = Server.basic_response_server(request_timeout=3) data = b'bananadine' - with server as address: sock = socket.socket() sock.connect(address) time.sleep(1.5) sock.sendall(data) sock.close() - assert server.handler_results[0] == data def test_server_finishes_on_error(self): @@ -151,16 +140,16 @@ class TestTestServer: assert len(server.handler_results) == 0 - # if the server thread fails to finish, the test suite will hang - # and get killed by the jenkins timeout. + # if the server thread fails to finish, the test suite will hang + # and get killed by the jenkins timeout. def test_server_finishes_when_no_connections(self): """the server thread exits even if there are no connections""" server = Server.basic_response_server() with server: pass - assert len(server.handler_results) == 0 - # if the server thread fails to finish, the test suite will hang - # and get killed by the jenkins timeout. + +# if the server thread fails to finish, the test suite will hang +# and get killed by the jenkins timeout. diff --git a/tests/test_utils.py b/tests/test_utils.py index 59b0b0ef..9e1c44d1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - import os import copy import filecmp @@ -8,33 +7,44 @@ import zipfile from collections import deque import pytest -from requests import compat -from requests.cookies import RequestsCookieJar -from requests.structures import CaseInsensitiveDict -from requests.utils import ( - address_in_network, dotted_netmask, extract_zipped_paths, - get_auth_from_url, _parse_content_type_header, get_encoding_from_headers, - get_encodings_from_content, get_environ_proxies, - guess_filename, guess_json_utf, is_ipv4_address, - is_valid_cidr, iter_slices, parse_dict_header, - parse_header_links, prepend_scheme_if_needed, - requote_uri, select_proxy, should_bypass_proxies, super_len, - to_key_val_list, to_native_string, - unquote_header_value, unquote_unreserved, - urldefragauth, add_dict_to_cookiejar, set_environ) -from requests._internal_utils import unicode_is_ascii +from requests3 import basics +from requests3.cookies import RequestsCookieJar +from requests3.structures import CaseInsensitiveDict +from requests3.utils import ( + address_in_network, + dotted_netmask, + get_auth_from_url, + get_encoding_from_headers, + get_encodings_from_content, + get_environ_proxies, + guess_filename, + guess_json_utf, + is_ipv4_address, + is_valid_cidr, + iter_slices, + parse_dict_header, + parse_header_links, + prepend_scheme_if_needed, + requote_uri, + select_proxy, + should_bypass_proxies, + super_len, + to_key_val_list, + unquote_header_value, + unquote_unreserved, + urldefragauth, + add_dict_to_cookiejar, + set_environ, +) +from requests3._internal_utils import unicode_is_ascii -from .compat import StringIO, cStringIO +from .compat import StringIO class TestSuperLen: - @pytest.mark.parametrize( - 'stream, value', ( - (StringIO.StringIO, 'Test'), - (BytesIO, b'Test'), - pytest.mark.skipif('cStringIO is None')((cStringIO, 'Test')), - )) + "stream, value", ((StringIO.StringIO, "Test"), (BytesIO, b"Test")) + ) def test_io_streams(self, stream, value): """Ensures that we properly deal with different kinds of IO streams.""" assert super_len(stream()) == 0 @@ -43,12 +53,13 @@ class TestSuperLen: def test_super_len_correctly_calculates_len_of_partially_read_file(self): """Ensure that we handle partially consumed file like objects.""" s = StringIO.StringIO() - s.write('foobarbogus') + s.write("foobarbogus") assert super_len(s) == 0 - @pytest.mark.parametrize('error', [IOError, OSError]) + @pytest.mark.parametrize("error", [IOError, OSError]) def test_super_len_handles_files_raising_weird_errors_in_tell(self, error): """If tell() raises errors, assume the cursor is at position zero.""" + class BoomFile(object): def __len__(self): return 5 @@ -58,9 +69,10 @@ class TestSuperLen: assert super_len(BoomFile()) == 0 - @pytest.mark.parametrize('error', [IOError, OSError]) + @pytest.mark.parametrize("error", [IOError, OSError]) def test_super_len_tell_ioerror(self, error): """Ensure that if tell gives an IOError super_len doesn't fail""" + class NoLenBoomFile(object): def tell(self): raise error() @@ -71,22 +83,18 @@ class TestSuperLen: assert super_len(NoLenBoomFile()) == 0 def test_string(self): - assert super_len('Test') == 4 + assert super_len("Test") == 4 - @pytest.mark.parametrize( - 'mode, warnings_num', ( - ('r', 1), - ('rb', 0), - )) + @pytest.mark.parametrize("mode, warnings_num", (("r", 1), ("rb", 0))) def test_file(self, tmpdir, mode, warnings_num, recwarn): - file_obj = tmpdir.join('test.txt') - file_obj.write('Test') + file_obj = tmpdir.join("test.txt") + file_obj.write("Test") with file_obj.open(mode) as fd: assert super_len(fd) == 4 assert len(recwarn) == warnings_num def test_super_len_with__len__(self): - foo = [1,2,3,4] + foo = [1, 2, 3, 4] len_foo = super_len(foo) assert len_foo == 4 @@ -98,13 +106,13 @@ class TestSuperLen: assert super_len(LenFile()) == 5 def test_super_len_with_tell(self): - foo = StringIO.StringIO('12345') + foo = StringIO.StringIO("12345") assert super_len(foo) == 5 foo.read(2) assert super_len(foo) == 3 def test_super_len_with_fileno(self): - with open(__file__, 'rb') as f: + with open(__file__, "rb") as f: length = super_len(f) file_data = f.read() assert length == len(file_data) @@ -115,37 +123,39 @@ class TestSuperLen: class TestToKeyValList: - @pytest.mark.parametrize( - 'value, expected', ( - ([('key', 'val')], [('key', 'val')]), - ((('key', 'val'), ), [('key', 'val')]), - ({'key': 'val'}, [('key', 'val')]), - (None, None) - )) + "value, expected", + ( + ([("key", "val")], [("key", "val")]), + ((("key", "val"),), [("key", "val")]), + ({"key": "val"}, [("key", "val")]), + (None, None), + ), + ) def test_valid(self, value, expected): assert to_key_val_list(value) == expected def test_invalid(self): with pytest.raises(ValueError): - to_key_val_list('string') + to_key_val_list("string") class TestUnquoteHeaderValue: - @pytest.mark.parametrize( - 'value, expected', ( + "value, expected", + ( (None, None), - ('Test', 'Test'), - ('"Test"', 'Test'), - ('"Test\\\\"', 'Test\\'), - ('"\\\\Comp\\Res"', '\\Comp\\Res'), - )) + ("Test", "Test"), + ('"Test"', "Test"), + ('"Test\\\\"', "Test\\"), + ('"\\\\Comp\\Res"', "\\Comp\\Res"), + ), + ) def test_valid(self, value, expected): assert unquote_header_value(value) == expected def test_is_filename(self): - assert unquote_header_value('"\\\\Comp\\Res"', True) == '\\\\Comp\\Res' + assert unquote_header_value('"\\\\Comp\\Res"', True) == "\\\\Comp\\Res" class TestGetEnvironProxies: @@ -153,131 +163,123 @@ class TestGetEnvironProxies: in no_proxy variable. """ - @pytest.fixture(autouse=True, params=['no_proxy', 'NO_PROXY']) + @pytest.fixture(autouse=True, params=["no_proxy", "NO_PROXY"]) def no_proxy(self, request, monkeypatch): - monkeypatch.setenv(request.param, '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') + monkeypatch.setenv( + request.param, "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1" + ) @pytest.mark.parametrize( - 'url', ( - 'http://192.168.0.1:5000/', - 'http://192.168.0.1/', - 'http://172.16.1.1/', - 'http://172.16.1.1:5000/', - 'http://localhost.localdomain:5000/v1.0/', - )) + "url", + ( + "http://192.168.0.1:5000/", + "http://192.168.0.1/", + "http://172.16.1.1/", + "http://172.16.1.1:5000/", + "http://localhost.localdomain:5000/v1.0/", + ), + ) def test_bypass(self, url): assert get_environ_proxies(url, no_proxy=None) == {} @pytest.mark.parametrize( - 'url', ( - 'http://192.168.1.1:5000/', - 'http://192.168.1.1/', - 'http://www.requests.com/', - )) + "url", + ("http://192.168.1.1:5000/", "http://192.168.1.1/", "http://www.requests.com/"), + ) def test_not_bypass(self, url): assert get_environ_proxies(url, no_proxy=None) != {} @pytest.mark.parametrize( - 'url', ( - 'http://192.168.1.1:5000/', - 'http://192.168.1.1/', - 'http://www.requests.com/', - )) + "url", + ("http://192.168.1.1:5000/", "http://192.168.1.1/", "http://www.requests.com/"), + ) def test_bypass_no_proxy_keyword(self, url): - no_proxy = '192.168.1.1,requests.com' + no_proxy = "192.168.1.1,requests.com" assert get_environ_proxies(url, no_proxy=no_proxy) == {} @pytest.mark.parametrize( - 'url', ( - 'http://192.168.0.1:5000/', - 'http://192.168.0.1/', - 'http://172.16.1.1/', - 'http://172.16.1.1:5000/', - 'http://localhost.localdomain:5000/v1.0/', - )) + "url", + ( + "http://192.168.0.1:5000/", + "http://192.168.0.1/", + "http://172.16.1.1/", + "http://172.16.1.1:5000/", + "http://localhost.localdomain:5000/v1.0/", + ), + ) def test_not_bypass_no_proxy_keyword(self, url, monkeypatch): # This is testing that the 'no_proxy' argument overrides the # environment variable 'no_proxy' - monkeypatch.setenv('http_proxy', 'http://proxy.example.com:3128/') - no_proxy = '192.168.1.1,requests.com' + monkeypatch.setenv("http_proxy", "http://proxy.example.com:3128/") + no_proxy = "192.168.1.1,requests.com" assert get_environ_proxies(url, no_proxy=no_proxy) != {} class TestIsIPv4Address: - def test_valid(self): - assert is_ipv4_address('8.8.8.8') + assert is_ipv4_address("8.8.8.8") - @pytest.mark.parametrize('value', ('8.8.8.8.8', 'localhost.localdomain')) + @pytest.mark.parametrize("value", ("8.8.8.8.8", "localhost.localdomain")) def test_invalid(self, value): assert not is_ipv4_address(value) class TestIsValidCIDR: - def test_valid(self): - assert is_valid_cidr('192.168.1.0/24') + assert is_valid_cidr("192.168.1.0/24") @pytest.mark.parametrize( - 'value', ( - '8.8.8.8', - '192.168.1.0/a', - '192.168.1.0/128', - '192.168.1.0/-1', - '192.168.1.999/24', - )) + "value", + ( + "8.8.8.8", + "192.168.1.0/a", + "192.168.1.0/128", + "192.168.1.0/-1", + "192.168.1.999/24", + ), + ) def test_invalid(self, value): assert not is_valid_cidr(value) class TestAddressInNetwork: - def test_valid(self): - assert address_in_network('192.168.1.1', '192.168.1.0/24') + assert address_in_network("192.168.1.1", "192.168.1.0/24") def test_invalid(self): - assert not address_in_network('172.16.0.1', '192.168.1.0/24') + assert not address_in_network("172.16.0.1", "192.168.1.0/24") class TestGuessFilename: - - @pytest.mark.parametrize( - 'value', (1, type('Fake', (object,), {'name': 1})()), - ) + @pytest.mark.parametrize("value", (1, type("Fake", (object,), {"name": 1})())) def test_guess_filename_invalid(self, value): assert guess_filename(value) is None @pytest.mark.parametrize( - 'value, expected_type', ( - (b'value', compat.bytes), - (b'value'.decode('utf-8'), compat.str) - )) + "value, expected_type", + ((b"value", basics.bytes), (b"value".decode("utf-8"), basics.str)), + ) def test_guess_filename_valid(self, value, expected_type): - obj = type('Fake', (object,), {'name': value})() + obj = type("Fake", (object,), {"name": value})() result = guess_filename(obj) assert result == value assert isinstance(result, expected_type) class TestExtractZippedPaths: - @pytest.mark.parametrize( - 'path', ( - '/', - __file__, - pytest.__file__, - '/etc/invalid/location', - )) + "path", ("/", __file__, pytest.__file__, "/etc/invalid/location") + ) def test_unzipped_paths_unchanged(self, path): assert path == extract_zipped_paths(path) def test_zipped_paths_extracted(self, tmpdir): - zipped_py = tmpdir.join('test.zip') - with zipfile.ZipFile(zipped_py.strpath, 'w') as f: + zipped_py = tmpdir.join("test.zip") + with zipfile.ZipFile(zipped_py.strpath, "w") as f: f.write(__file__) _, name = os.path.splitdrive(__file__) - zipped_path = os.path.join(zipped_py.strpath, name.lstrip(r'\/')) + zipped_path = os.path.join(zipped_py.strpath, name.lstrip(r"\/")) extracted_path = extract_zipped_paths(zipped_path) assert extracted_path != zipped_path @@ -286,261 +288,221 @@ class TestExtractZippedPaths: class TestContentEncodingDetection: - def test_none(self): - encodings = get_encodings_from_content('') + encodings = get_encodings_from_content("") assert not len(encodings) @pytest.mark.parametrize( - 'content', ( - # HTML5 meta charset attribute + "content", + ( '', - # HTML4 pragma directive '', - # XHTML 1.x served with text/html MIME type '', - # XHTML 1.x served as XML '', - )) + ), + # HTML5 meta charset attribute + # HTML4 pragma directive + # XHTML 1.x served with text/html MIME type + # XHTML 1.x served as XML + ) def test_pragmas(self, content): encodings = get_encodings_from_content(content) assert len(encodings) == 1 - assert encodings[0] == 'UTF-8' + assert encodings[0] == "UTF-8" def test_precedence(self): - content = ''' + content = """ - '''.strip() - assert get_encodings_from_content(content) == ['HTML5', 'HTML4', 'XML'] + """.strip() + assert get_encodings_from_content(content) == ["HTML5", "HTML4", "XML"] class TestGuessJSONUTF: - @pytest.mark.parametrize( - 'encoding', ( - 'utf-32', 'utf-8-sig', 'utf-16', 'utf-8', 'utf-16-be', 'utf-16-le', - 'utf-32-be', 'utf-32-le' - )) + "encoding", + ( + "utf-32", + "utf-8-sig", + "utf-16", + "utf-8", + "utf-16-be", + "utf-16-le", + "utf-32-be", + "utf-32-le", + ), + ) def test_encoded(self, encoding): - data = '{}'.encode(encoding) + data = "{}".encode(encoding) assert guess_json_utf(data) == encoding def test_bad_utf_like_encoding(self): - assert guess_json_utf(b'\x00\x00\x00\x00') is None + assert guess_json_utf(b"\x00\x00\x00\x00") is None @pytest.mark.parametrize( - ('encoding', 'expected'), ( - ('utf-16-be', 'utf-16'), - ('utf-16-le', 'utf-16'), - ('utf-32-be', 'utf-32'), - ('utf-32-le', 'utf-32') - )) + ("encoding", "expected"), + ( + ("utf-16-be", "utf-16"), + ("utf-16-le", "utf-16"), + ("utf-32-be", "utf-32"), + ("utf-32-le", "utf-32"), + ), + ) def test_guess_by_bom(self, encoding, expected): - data = u'\ufeff{}'.encode(encoding) + data = u"\ufeff{}".encode(encoding) assert guess_json_utf(data) == expected USER = PASSWORD = "%!*'();:@&=+$,/?#[] " -ENCODED_USER = compat.quote(USER, '') -ENCODED_PASSWORD = compat.quote(PASSWORD, '') +ENCODED_USER = basics.quote(USER, "") +ENCODED_PASSWORD = basics.quote(PASSWORD, "") @pytest.mark.parametrize( - 'url, auth', ( + "url, auth", + ( ( - 'http://' + ENCODED_USER + ':' + ENCODED_PASSWORD + '@' + - 'request.com/url.html#test', - (USER, PASSWORD) + "http://" + + ENCODED_USER + + ":" + + ENCODED_PASSWORD + + "@" + + "request.com/url.html#test", + (USER, PASSWORD), + ), + ("http://user:pass@complex.url.com/path?query=yes", ("user", "pass")), + ( + "http://user:pass%20pass@complex.url.com/path?query=yes", + ("user", "pass pass"), + ), + ("http://user:pass pass@complex.url.com/path?query=yes", ("user", "pass pass")), + ( + "http://user%25user:pass@complex.url.com/path?query=yes", + ("user%user", "pass"), ), ( - 'http://user:pass@complex.url.com/path?query=yes', - ('user', 'pass') + "http://user:pass%23pass@complex.url.com/path?query=yes", + ("user", "pass#pass"), ), - ( - 'http://user:pass%20pass@complex.url.com/path?query=yes', - ('user', 'pass pass') - ), - ( - 'http://user:pass pass@complex.url.com/path?query=yes', - ('user', 'pass pass') - ), - ( - 'http://user%25user:pass@complex.url.com/path?query=yes', - ('user%user', 'pass') - ), - ( - 'http://user:pass%23pass@complex.url.com/path?query=yes', - ('user', 'pass#pass') - ), - ( - 'http://complex.url.com/path?query=yes', - ('', '') - ), - )) + ("http://complex.url.com/path?query=yes", ("", "")), + ), +) def test_get_auth_from_url(url, auth): assert get_auth_from_url(url) == auth @pytest.mark.parametrize( - 'uri, expected', ( + "uri, expected", + ( ( - # Ensure requoting doesn't break expectations - 'http://example.com/fiz?buz=%25ppicture', - 'http://example.com/fiz?buz=%25ppicture', + "http://example.com/fiz?buz=%25ppicture", + "http://example.com/fiz?buz=%25ppicture", ), ( - # Ensure we handle unquoted percent signs in redirects - 'http://example.com/fiz?buz=%ppicture', - 'http://example.com/fiz?buz=%25ppicture', + "http://example.com/fiz?buz=%ppicture", + "http://example.com/fiz?buz=%25ppicture", ), - )) + ), + # Ensure requoting doesn't break expectations + # Ensure we handle unquoted percent signs in redirects +) def test_requote_uri_with_unquoted_percents(uri, expected): """See: https://github.com/requests/requests/issues/2356""" assert requote_uri(uri) == expected @pytest.mark.parametrize( - 'uri, expected', ( - ( - # Illegal bytes - 'http://example.com/?a=%--', - 'http://example.com/?a=%--', - ), - ( - # Reserved characters - 'http://example.com/?a=%300', - 'http://example.com/?a=00', - ) - )) + "uri, expected", + ( + ("http://example.com/?a=%--", "http://example.com/?a=%--"), + ("http://example.com/?a=%300", "http://example.com/?a=00"), + ), + # Illegal bytes + # Reserved characters +) def test_unquote_unreserved(uri, expected): assert unquote_unreserved(uri) == expected @pytest.mark.parametrize( - 'mask, expected', ( - (8, '255.0.0.0'), - (24, '255.255.255.0'), - (25, '255.255.255.128'), - )) + "mask, expected", ((8, "255.0.0.0"), (24, "255.255.255.0"), (25, "255.255.255.128")) +) def test_dotted_netmask(mask, expected): assert dotted_netmask(mask) == expected -http_proxies = {'http': 'http://http.proxy', - 'http://some.host': 'http://some.host.proxy'} -all_proxies = {'all': 'socks5://http.proxy', - 'all://some.host': 'socks5://some.host.proxy'} -mixed_proxies = {'http': 'http://http.proxy', - 'http://some.host': 'http://some.host.proxy', - 'all': 'socks5://http.proxy'} +http_proxies = { + "http": "http://http.proxy", + "http://some.host": "http://some.host.proxy", +} +all_proxies = { + "all": "socks5://http.proxy", + "all://some.host": "socks5://some.host.proxy", +} +mixed_proxies = { + "http": "http://http.proxy", + "http://some.host": "http://some.host.proxy", + "all": "socks5://http.proxy", +} + + @pytest.mark.parametrize( - 'url, expected, proxies', ( - ('hTTp://u:p@Some.Host/path', 'http://some.host.proxy', http_proxies), - ('hTTp://u:p@Other.Host/path', 'http://http.proxy', http_proxies), - ('hTTp:///path', 'http://http.proxy', http_proxies), - ('hTTps://Other.Host', None, http_proxies), - ('file:///etc/motd', None, http_proxies), - - ('hTTp://u:p@Some.Host/path', 'socks5://some.host.proxy', all_proxies), - ('hTTp://u:p@Other.Host/path', 'socks5://http.proxy', all_proxies), - ('hTTp:///path', 'socks5://http.proxy', all_proxies), - ('hTTps://Other.Host', 'socks5://http.proxy', all_proxies), - - ('http://u:p@other.host/path', 'http://http.proxy', mixed_proxies), - ('http://u:p@some.host/path', 'http://some.host.proxy', mixed_proxies), - ('https://u:p@other.host/path', 'socks5://http.proxy', mixed_proxies), - ('https://u:p@some.host/path', 'socks5://http.proxy', mixed_proxies), - ('https://', 'socks5://http.proxy', mixed_proxies), - # XXX: unsure whether this is reasonable behavior - ('file:///etc/motd', 'socks5://http.proxy', all_proxies), - )) + "url, expected, proxies", + ( + ("hTTp://u:p@Some.Host/path", "http://some.host.proxy", http_proxies), + ("hTTp://u:p@Other.Host/path", "http://http.proxy", http_proxies), + ("hTTp:///path", "http://http.proxy", http_proxies), + ("hTTps://Other.Host", None, http_proxies), + ("file:///etc/motd", None, http_proxies), + ("hTTp://u:p@Some.Host/path", "socks5://some.host.proxy", all_proxies), + ("hTTp://u:p@Other.Host/path", "socks5://http.proxy", all_proxies), + ("hTTp:///path", "socks5://http.proxy", all_proxies), + ("hTTps://Other.Host", "socks5://http.proxy", all_proxies), + ("http://u:p@other.host/path", "http://http.proxy", mixed_proxies), + ("http://u:p@some.host/path", "http://some.host.proxy", mixed_proxies), + ("https://u:p@other.host/path", "socks5://http.proxy", mixed_proxies), + ("https://u:p@some.host/path", "socks5://http.proxy", mixed_proxies), + ("https://", "socks5://http.proxy", mixed_proxies), + ("file:///etc/motd", "socks5://http.proxy", all_proxies), + ), + # XXX: unsure whether this is reasonable behavior +) def test_select_proxies(url, expected, proxies): """Make sure we can select per-host proxies correctly.""" assert select_proxy(url, proxies) == expected @pytest.mark.parametrize( - 'value, expected', ( - ('foo="is a fish", bar="as well"', {'foo': 'is a fish', 'bar': 'as well'}), - ('key_without_value', {'key_without_value': None}) - )) + "value, expected", + ( + ('foo="is a fish", bar="as well"', {"foo": "is a fish", "bar": "as well"}), + ("key_without_value", {"key_without_value": None}), + ), +) def test_parse_dict_header(value, expected): assert parse_dict_header(value) == expected @pytest.mark.parametrize( - 'value, expected', ( + "value, expected", + ( + (CaseInsensitiveDict(), None), ( - 'application/xml', - ('application/xml', {}) + CaseInsensitiveDict({"content-type": "application/json; charset=utf-8"}), + "utf-8", ), - ( - 'application/json ; charset=utf-8', - ('application/json', {'charset': 'utf-8'}) - ), - ( - 'application/json ; Charset=utf-8', - ('application/json', {'charset': 'utf-8'}) - ), - ( - 'text/plain', - ('text/plain', {}) - ), - ( - 'multipart/form-data; boundary = something ; boundary2=\'something_else\' ; no_equals ', - ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) - ), - ( - 'multipart/form-data; boundary = something ; boundary2="something_else" ; no_equals ', - ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) - ), - ( - 'multipart/form-data; boundary = something ; \'boundary2=something_else\' ; no_equals ', - ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) - ), - ( - 'multipart/form-data; boundary = something ; "boundary2=something_else" ; no_equals ', - ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) - ), - ( - 'application/json ; ; ', - ('application/json', {}) - ) - )) -def test__parse_content_type_header(value, expected): - assert _parse_content_type_header(value) == expected - - -@pytest.mark.parametrize( - 'value, expected', ( - ( - CaseInsensitiveDict(), - None - ), - ( - CaseInsensitiveDict({'content-type': 'application/json; charset=utf-8'}), - 'utf-8' - ), - ( - CaseInsensitiveDict({'content-type': 'text/plain'}), - 'ISO-8859-1' - ), - )) + (CaseInsensitiveDict({"content-type": "text/plain"}), "ISO-8859-1"), + ), +) def test_get_encoding_from_headers(value, expected): assert get_encoding_from_headers(value) == expected @pytest.mark.parametrize( - 'value, length', ( - ('', 0), - ('T', 1), - ('Test', 4), - ('Cont', 0), - ('Other', -5), - ('Content', None), - )) + "value, length", + (("", 0), ("T", 1), ("Test", 4), ("Cont", 0), ("Other", -5), ("Content", None)), +) def test_iter_slices(value, length): if length is None or (length <= 0 and len(value) > 0): # Reads all content at once @@ -550,182 +512,146 @@ def test_iter_slices(value, length): @pytest.mark.parametrize( - 'value, expected', ( + "value, expected", + ( ( '; rel=front; type="image/jpeg"', - [{'url': 'http:/.../front.jpeg', 'rel': 'front', 'type': 'image/jpeg'}] - ), - ( - '', - [{'url': 'http:/.../front.jpeg'}] - ), - ( - ';', - [{'url': 'http:/.../front.jpeg'}] + [{"url": "http:/.../front.jpeg", "rel": "front", "type": "image/jpeg"}], ), + ("", [{"url": "http:/.../front.jpeg"}]), + (";", [{"url": "http:/.../front.jpeg"}]), ( '; type="image/jpeg",;', [ - {'url': 'http:/.../front.jpeg', 'type': 'image/jpeg'}, - {'url': 'http://.../back.jpeg'} - ] + {"url": "http:/.../front.jpeg", "type": "image/jpeg"}, + {"url": "http://.../back.jpeg"}, + ], ), - ( - '', - [] - ), - )) + ("", []), + ), +) def test_parse_header_links(value, expected): assert parse_header_links(value) == expected @pytest.mark.parametrize( - 'value, expected', ( - ('example.com/path', 'http://example.com/path'), - ('//example.com/path', 'http://example.com/path'), - )) + "value, expected", + ( + ("example.com/path", "http://example.com/path"), + ("//example.com/path", "http://example.com/path"), + ), +) def test_prepend_scheme_if_needed(value, expected): - assert prepend_scheme_if_needed(value, 'http') == expected + assert prepend_scheme_if_needed(value, "http") == expected @pytest.mark.parametrize( - 'value, expected', ( - ('T', 'T'), - (b'T', 'T'), - (u'T', 'T'), - )) -def test_to_native_string(value, expected): - assert to_native_string(value) == expected - - -@pytest.mark.parametrize( - 'url, expected', ( - ('http://u:p@example.com/path?a=1#test', 'http://example.com/path?a=1'), - ('http://example.com/path', 'http://example.com/path'), - ('//u:p@example.com/path', '//example.com/path'), - ('//example.com/path', '//example.com/path'), - ('example.com/path', '//example.com/path'), - ('scheme:u:p@example.com/path', 'scheme://example.com/path'), - )) + "url, expected", + ( + ("http://u:p@example.com/path?a=1#test", "http://example.com/path?a=1"), + ("http://example.com/path", "http://example.com/path"), + ("//u:p@example.com/path", "//example.com/path"), + ("//example.com/path", "//example.com/path"), + ("example.com/path", "//example.com/path"), + ("scheme:u:p@example.com/path", "scheme://example.com/path"), + ), +) def test_urldefragauth(url, expected): assert urldefragauth(url) == expected @pytest.mark.parametrize( - 'url, expected', ( - ('http://192.168.0.1:5000/', True), - ('http://192.168.0.1/', True), - ('http://172.16.1.1/', True), - ('http://172.16.1.1:5000/', True), - ('http://localhost.localdomain:5000/v1.0/', True), - ('http://google.com:6000/', True), - ('http://172.16.1.12/', False), - ('http://172.16.1.12:5000/', False), - ('http://google.com:5000/v1.0/', False), - ('file:///some/path/on/disk', True), - )) + "url, expected", + ( + ("http://192.168.0.1:5000/", True), + ("http://192.168.0.1/", True), + ("http://172.16.1.1/", True), + ("http://172.16.1.1:5000/", True), + ("http://localhost.localdomain:5000/v1.0/", True), + ("http://172.16.1.12/", False), + ("http://172.16.1.12:5000/", False), + ("http://google.com:5000/v1.0/", False), + ), +) def test_should_bypass_proxies(url, expected, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not """ - monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000') - monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000') + monkeypatch.setenv( + "no_proxy", "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1" + ) + monkeypatch.setenv( + "NO_PROXY", "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1" + ) assert should_bypass_proxies(url, no_proxy=None) == expected @pytest.mark.parametrize( - 'url, expected', ( - ('http://172.16.1.1/', '172.16.1.1'), - ('http://172.16.1.1:5000/', '172.16.1.1'), - ('http://user:pass@172.16.1.1', '172.16.1.1'), - ('http://user:pass@172.16.1.1:5000', '172.16.1.1'), - ('http://hostname/', 'hostname'), - ('http://hostname:5000/', 'hostname'), - ('http://user:pass@hostname', 'hostname'), - ('http://user:pass@hostname:5000', 'hostname'), - )) -def test_should_bypass_proxies_pass_only_hostname(url, expected, mocker): - """The proxy_bypass function should be called with a hostname or IP without - a port number or auth credentials. - """ - proxy_bypass = mocker.patch('requests.utils.proxy_bypass') - should_bypass_proxies(url, no_proxy=None) - proxy_bypass.assert_called_once_with(expected) - - -@pytest.mark.parametrize( - 'cookiejar', ( - compat.cookielib.CookieJar(), - RequestsCookieJar() - )) + "cookiejar", (basics.cookielib.CookieJar(), RequestsCookieJar()) +) def test_add_dict_to_cookiejar(cookiejar): """Ensure add_dict_to_cookiejar works for non-RequestsCookieJar CookieJars """ - cookiedict = {'test': 'cookies', - 'good': 'cookies'} + cookiedict = {"test": "cookies", "good": "cookies"} cj = add_dict_to_cookiejar(cookiejar, cookiedict) cookies = {cookie.name: cookie.value for cookie in cj} assert cookiedict == cookies @pytest.mark.parametrize( - 'value, expected', ( - (u'test', True), - (u'æíöû', False), - (u'ジェーピーニック', False), - ) + "value, expected", ((u"test", True), (u"æíöû", False), (u"ジェーピーニック", False)) ) def test_unicode_is_ascii(value, expected): assert unicode_is_ascii(value) is expected @pytest.mark.parametrize( - 'url, expected', ( - ('http://192.168.0.1:5000/', True), - ('http://192.168.0.1/', True), - ('http://172.16.1.1/', True), - ('http://172.16.1.1:5000/', True), - ('http://localhost.localdomain:5000/v1.0/', True), - ('http://172.16.1.12/', False), - ('http://172.16.1.12:5000/', False), - ('http://google.com:5000/v1.0/', False), - )) -def test_should_bypass_proxies_no_proxy( - url, expected, monkeypatch): + "url, expected", + ( + ("http://192.168.0.1:5000/", True), + ("http://192.168.0.1/", True), + ("http://172.16.1.1/", True), + ("http://172.16.1.1:5000/", True), + ("http://localhost.localdomain:5000/v1.0/", True), + ("http://172.16.1.12/", False), + ("http://172.16.1.12:5000/", False), + ("http://google.com:5000/v1.0/", False), + ), +) +def test_should_bypass_proxies_no_proxy(url, expected, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not using the 'no_proxy' argument """ - no_proxy = '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1' + no_proxy = "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1" # Test 'no_proxy' argument assert should_bypass_proxies(url, no_proxy=no_proxy) == expected -@pytest.mark.skipif(os.name != 'nt', reason='Test only on Windows') +@pytest.mark.skipif(os.name != "nt", reason="Test only on Windows") @pytest.mark.parametrize( - 'url, expected, override', ( - ('http://192.168.0.1:5000/', True, None), - ('http://192.168.0.1/', True, None), - ('http://172.16.1.1/', True, None), - ('http://172.16.1.1:5000/', True, None), - ('http://localhost.localdomain:5000/v1.0/', True, None), - ('http://172.16.1.22/', False, None), - ('http://172.16.1.22:5000/', False, None), - ('http://google.com:5000/v1.0/', False, None), - ('http://mylocalhostname:5000/v1.0/', True, ''), - ('http://192.168.0.1/', False, ''), - )) -def test_should_bypass_proxies_win_registry(url, expected, override, - monkeypatch): + "url, expected, override", + ( + ("http://192.168.0.1:5000/", True, None), + ("http://192.168.0.1/", True, None), + ("http://172.16.1.1/", True, None), + ("http://172.16.1.1:5000/", True, None), + ("http://localhost.localdomain:5000/v1.0/", True, None), + ("http://172.16.1.22/", False, None), + ("http://172.16.1.22:5000/", False, None), + ("http://google.com:5000/v1.0/", False, None), + ("http://mylocalhostname:5000/v1.0/", True, ""), + ("http://192.168.0.1/", False, ""), + ), +) +def test_should_bypass_proxies_win_registry(url, expected, override, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not with Windows registry settings """ if override is None: - override = '192.168.*;127.0.0.1;localhost.localdomain;172.16.1.1' - if compat.is_py3: - import winreg - else: - import _winreg as winreg + override = "192.168.*;127.0.0.1;localhost.localdomain;172.16.1.1" + + import winreg class RegHandle: def Close(self): @@ -739,36 +665,36 @@ def test_should_bypass_proxies_win_registry(url, expected, override, def QueryValueEx(key, value_name): if key is ie_settings: - if value_name == 'ProxyEnable': - # this could be a string (REG_SZ) or a 32-bit number (REG_DWORD) - proxyEnableValues.rotate() - return [proxyEnableValues[0]] - elif value_name == 'ProxyOverride': + if value_name == "ProxyEnable": + return [1] + + elif value_name == "ProxyOverride": return [override] - monkeypatch.setenv('http_proxy', '') - monkeypatch.setenv('https_proxy', '') - monkeypatch.setenv('ftp_proxy', '') - monkeypatch.setenv('no_proxy', '') - monkeypatch.setenv('NO_PROXY', '') - monkeypatch.setattr(winreg, 'OpenKey', OpenKey) - monkeypatch.setattr(winreg, 'QueryValueEx', QueryValueEx) + monkeypatch.setenv("http_proxy", "") + monkeypatch.setenv("https_proxy", "") + monkeypatch.setenv("ftp_proxy", "") + monkeypatch.setenv("no_proxy", "") + monkeypatch.setenv("NO_PROXY", "") + monkeypatch.setattr(winreg, "OpenKey", OpenKey) + monkeypatch.setattr(winreg, "QueryValueEx", QueryValueEx) assert should_bypass_proxies(url, None) == expected @pytest.mark.parametrize( - 'env_name, value', ( - ('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain'), - ('no_proxy', None), - ('a_new_key', '192.168.0.0/24,127.0.0.1,localhost.localdomain'), - ('a_new_key', None), - )) + "env_name, value", + ( + ("no_proxy", "192.168.0.0/24,127.0.0.1,localhost.localdomain"), + ("no_proxy", None), + ("a_new_key", "192.168.0.0/24,127.0.0.1,localhost.localdomain"), + ("a_new_key", None), + ), +) def test_set_environ(env_name, value): """Tests set_environ will set environ values and will restore the environ.""" environ_copy = copy.deepcopy(os.environ) with set_environ(env_name, value): assert os.environ.get(env_name) == value - assert os.environ == environ_copy @@ -776,7 +702,7 @@ def test_set_environ_raises_exception(): """Tests set_environ will raise exceptions in context when the value parameter is None.""" with pytest.raises(Exception) as exception: - with set_environ('test1', None): - raise Exception('Expected exception') + with set_environ("test1", None): + raise Exception("Expected exception") - assert 'Expected exception' in str(exception.value) + assert "Expected exception" in str(exception.value) diff --git a/tests/testserver/server.py b/tests/testserver/server.py index 6a1dcaa5..c766b32a 100644 --- a/tests/testserver/server.py +++ b/tests/testserver/server.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - import threading import socket import select @@ -8,7 +7,6 @@ import select def consume_socket_content(sock, timeout=0.5): chunks = 65536 content = b'' - while True: more_to_read = select.select([sock], [], [], timeout)[0] if not more_to_read: @@ -19,7 +17,6 @@ def consume_socket_content(sock, timeout=0.5): break content += new_content - return content @@ -27,37 +24,40 @@ class Server(threading.Thread): """Dummy server using for unit testing""" WAIT_EVENT_TIMEOUT = 5 - def __init__(self, handler=None, host='localhost', port=0, requests_to_handle=1, wait_to_close_event=None): + def __init__( + self, + handler=None, + host='localhost', + port=0, + requests_to_handle=1, + wait_to_close_event=None, + ): super(Server, self).__init__() - self.handler = handler or consume_socket_content self.handler_results = [] - self.host = host self.port = port self.requests_to_handle = requests_to_handle - self.wait_to_close_event = wait_to_close_event self.ready_event = threading.Event() self.stop_event = threading.Event() @classmethod def text_response_server(cls, text, request_timeout=0.5, **kwargs): + def text_response_handler(sock): - request_content = consume_socket_content(sock, timeout=request_timeout) + request_content = consume_socket_content( + sock, timeout=request_timeout + ) sock.send(text.encode('utf-8')) - return request_content - return Server(text_response_handler, **kwargs) @classmethod def basic_response_server(cls, **kwargs): return cls.text_response_server( - "HTTP/1.1 200 OK\r\n" + - "Content-Length: 0\r\n\r\n", - **kwargs + "HTTP/1.1 200 OK\r\n" + "Content-Length: 0\r\n\r\n", **kwargs ) def run(self): @@ -67,11 +67,10 @@ class Server(threading.Thread): self.port = self.server_sock.getsockname()[1] self.ready_event.set() self._handle_requests() - if self.wait_to_close_event: self.wait_to_close_event.wait(self.WAIT_EVENT_TIMEOUT) finally: - self.ready_event.set() # just in case of exception + self.ready_event.set() # just in case of exception self._close_server_sock_ignore_errors() self.stop_event.set() @@ -94,16 +93,18 @@ class Server(threading.Thread): break handler_result = self.handler(sock) - self.handler_results.append(handler_result) def _accept_connection(self): try: - ready, _, _ = select.select([self.server_sock], [], [], self.WAIT_EVENT_TIMEOUT) + ready, _, _ = select.select( + [self.server_sock], [], [], self.WAIT_EVENT_TIMEOUT + ) if not ready: return None return self.server_sock.accept()[0] + except (select.error, socket.error): return None @@ -120,8 +121,7 @@ class Server(threading.Thread): # avoid server from waiting for event timeouts # if an exception is found in the main thread self.wait_to_close_event.set() - # ensure server thread doesn't get stuck waiting for connections self._close_server_sock_ignore_errors() self.join() - return False # allow exceptions to propagate + return False # allow exceptions to propagate diff --git a/tests/utils.py b/tests/utils.py index 9b797fd4..b19b3554 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - import contextlib import os @@ -14,6 +13,7 @@ def override_environ(**kwargs): os.environ[key] = value try: yield + finally: os.environ.clear() os.environ.update(save_env) diff --git a/tox.ini b/tox.ini index de68e90d..169309bb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py34,py35,py36,py37 +envlist = py36 [testenv]