diff --git a/AUTHORS.rst b/AUTHORS.rst index b87dc443..2b4494ba 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -178,3 +178,8 @@ Patches and Suggestions - Ryan Pineo (`@ryanpineo `_) - Ed Morley (`@edmorley `_) - Matt Liu (`@mlcrazy `_) +- Taylor Hoff (`@PrimordialHelios `_) +- Arthur Vigil (`@ahvigil `_) +- Nehal J Wani (`@nehaljwani `_) +- Demetrios Bairaktaris (`@DemetriosBairaktaris `_) +- Darren Dormer (`@ddormer `_) diff --git a/HISTORY.rst b/HISTORY.rst index 7f18ea36..db1d1f70 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,9 +8,17 @@ dev **Improvements** +- Warn user about possible slowdown when using cryptography version < 1.3.4 +- Check for invalid host in proxy URL, before forwarding request to adapter. + **Bugfixes** - Parsing empty ``Link`` headers with ``parse_header_links()`` no longer return one bogus entry +- Fixed issue where loading the default certificate bundle from a zip archive + would raise an ``IOError`` +- Fixed issue with unexpected ``ImportError`` on windows system which do not support ``winreg`` module +- DNS resolution in proxy bypass no longer includes the username and password in + the request. This also fixes the issue of DNS queries failing on macOS. 2.18.4 (2017-08-15) +++++++++++++++++++ diff --git a/README.rst b/README.rst index e0dc5e4f..c7b033be 100644 --- a/README.rst +++ b/README.rst @@ -77,7 +77,7 @@ Requests is ready for today's web. - ``.netrc`` Support - Chunked Requests -Requests officially supports Python 2.6–2.7 & 3.3–3.7, and runs great on PyPy. +Requests officially supports Python 2.6–2.7 & 3.4–3.6, and runs great on PyPy. Installation ------------ @@ -105,6 +105,6 @@ How to Contribute #. Write a test which shows that the bug was fixed or that the feature works as expected. #. Send a pull request and bug the maintainer until it gets merged and published. :) Make sure to add yourself to AUTHORS_. -.. _`the repository`: http://github.com/requests/requests +.. _`the repository`: https://github.com/requests/requests .. _AUTHORS: https://github.com/requests/requests/blob/master/AUTHORS.rst .. _Contributor Friendly: https://github.com/requests/requests/issues?direction=desc&labels=Contributor+Friendly&page=1&sort=updated&state=open diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html index c1ed8b89..52c82aa5 100644 --- a/docs/_templates/sidebarintro.html +++ b/docs/_templates/sidebarintro.html @@ -5,7 +5,7 @@

-

@@ -20,7 +20,7 @@

Stay Informed

Receive updates on new releases and upcoming projects.

-

@@ -48,9 +48,9 @@

-
  • Requests @ GitHub
    • -
  • Requests @ PyPI
    • -
  • Issue Tracker
    • +
  • Requests @ GitHub
    • +
  • Requests @ PyPI
    • +
  • Issue Tracker
  • Release History
    • diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index 1803db45..c2d824a4 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -4,7 +4,7 @@

    -

    @@ -25,7 +25,7 @@

    If you enjoy using this project, Say Thanks!

    -

    diff --git a/docs/api.rst b/docs/api.rst index ed61bb38..c3e00e54 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -109,17 +109,7 @@ Status Code Lookup .. autoclass:: requests.codes -:: - - >>> requests.codes['temporary_redirect'] - 307 - - >>> requests.codes.teapot - 418 - - >>> requests.codes['\o/'] - 200 - +.. automodule:: requests.status_codes Migrating to 1.x diff --git a/docs/community/out-there.rst b/docs/community/out-there.rst index 5ce5f79f..63e70169 100644 --- a/docs/community/out-there.rst +++ b/docs/community/out-there.rst @@ -18,7 +18,7 @@ Articles & Talks - `Python for the Web `_ teaches how to use Python to interact with the web, using Requests. - `Daniel Greenfeld's Review of Requests `_ - `My 'Python for Humans' talk `_ ( `audio `_ ) -- `Issac Kelly's 'Consuming Web APIs' talk `_ +- `Issac Kelly's 'Consuming Web APIs' talk `_ - `Blog post about Requests via Yum `_ - `Russian blog post introducing Requests `_ - `Sending JSON in Requests `_ diff --git a/docs/community/recommended.rst b/docs/community/recommended.rst index 0f652d54..88dcce8d 100644 --- a/docs/community/recommended.rst +++ b/docs/community/recommended.rst @@ -34,7 +34,7 @@ but do not belong in Requests proper. This library is actively maintained by members of the Requests core team, and reflects the functionality most requested by users within the community. -.. _Requests-Toolbelt: http://toolbelt.readthedocs.io/en/latest/index.html +.. _Requests-Toolbelt: https://toolbelt.readthedocs.io/en/latest/index.html Requests-Threads diff --git a/docs/conf.py b/docs/conf.py index 4bda98b0..503448d3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -376,4 +376,4 @@ epub_exclude_files = ['search.html'] # If false, no index is generated. #epub_use_index = True -intersphinx_mapping = {'urllib3': ('http://urllib3.readthedocs.io/en/latest', None)} +intersphinx_mapping = {'urllib3': ('https://urllib3.readthedocs.io/en/latest', None)} diff --git a/docs/dev/todo.rst b/docs/dev/todo.rst index 50b18155..707dea31 100644 --- a/docs/dev/todo.rst +++ b/docs/dev/todo.rst @@ -61,5 +61,5 @@ Requests currently supports the following versions of Python: Google AppEngine is not officially supported although support is available with the `Requests-Toolbelt`_. -.. _Requests-Toolbelt: http://toolbelt.readthedocs.io/ +.. _Requests-Toolbelt: https://toolbelt.readthedocs.io/ diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 613df205..1bad6435 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -287,7 +287,7 @@ system. For the sake of security we recommend upgrading certifi frequently! .. _HTTP persistent connection: https://en.wikipedia.org/wiki/HTTP_persistent_connection -.. _connection pooling: http://urllib3.readthedocs.io/en/latest/reference/index.html#module-urllib3.connectionpool +.. _connection pooling: https://urllib3.readthedocs.io/en/latest/reference/index.html#module-urllib3.connectionpool .. _certifi: http://certifi.io/ .. _Mozilla trust store: https://hg.mozilla.org/mozilla-central/raw-file/tip/security/nss/lib/ckfw/builtins/certdata.txt @@ -436,7 +436,7 @@ You can assign a hook function on a per-request basis by passing a ``{hook_name: callback_function}`` dictionary to the ``hooks`` request parameter:: - hooks=dict(response=print_url) + hooks={'response': print_url} That ``callback_function`` will receive a chunk of data as its first argument. @@ -452,12 +452,36 @@ If the callback function returns a value, it is assumed that it is to replace the data that was passed in. If the function doesn't return anything, nothing else is effected. +:: + + def record_hook(r, *args, **kwargs): + r.hook_called = True + return r + Let's print some request method arguments at runtime:: - >>> requests.get('http://httpbin.org', hooks=dict(response=print_url)) + >>> requests.get('http://httpbin.org', hooks={'response': print_url}) http://httpbin.org +You can add multiple hooks to a single request. Let's call two hooks at once:: + + >>> r = requests.get('http://httpbin.org', hooks={'response': [print_url, record_hook]}) + >>> r.hook_called + True + +You can also add hooks to a ``Session`` instance. Any hooks you add will then +be called on every request made to the session. For example:: + + >>> s = requests.Session() + >>> s.hooks['response'].append(print_url) + >>> s.get('http://httpbin.org') + http://httpbin.org + + +A ``Session`` can have multiple hooks, which will be called in the order +they are added. + .. _custom-auth: Custom Authentication @@ -633,7 +657,7 @@ When you receive a response, Requests makes a guess at the encoding to use for decoding the response when you access the :attr:`Response.text ` attribute. Requests will first check for an encoding in the HTTP header, and if none is present, will use `chardet -`_ to attempt to guess the encoding. +`_ to attempt to guess the encoding. The only time Requests will not do this is if no explicit charset is present in the HTTP headers **and** the ``Content-Type`` @@ -860,7 +884,7 @@ Link Headers Many HTTP APIs feature Link headers. They make APIs more self describing and discoverable. -GitHub uses these for `pagination `_ +GitHub uses these for `pagination `_ in their API, for example:: >>> url = 'https://api.github.com/users/kennethreitz/repos?page=1&per_page=10' diff --git a/docs/user/authentication.rst b/docs/user/authentication.rst index 8ffab504..411f79fd 100644 --- a/docs/user/authentication.rst +++ b/docs/user/authentication.rst @@ -136,11 +136,11 @@ Further examples can be found under the `Requests organization`_ and in the .. _OAuth: http://oauth.net/ .. _requests_oauthlib: https://github.com/requests/requests-oauthlib -.. _requests-oauthlib OAuth2 documentation: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html -.. _Web Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#web-application-flow -.. _Mobile Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#mobile-application-flow -.. _Legacy Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#legacy-application-flow -.. _Backend Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#backend-application-flow +.. _requests-oauthlib OAuth2 documentation: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html +.. _Web Application Flow: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#web-application-flow +.. _Mobile Application Flow: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#mobile-application-flow +.. _Legacy Application Flow: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#legacy-application-flow +.. _Backend Application Flow: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#backend-application-flow .. _Kerberos: https://github.com/requests/requests-kerberos .. _NTLM: https://github.com/requests/requests-ntlm .. _Requests organization: https://github.com/requests diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 7fe1e0d2..1a2c6fbf 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -110,7 +110,7 @@ using, and change it, using the ``r.encoding`` property:: If you change the encoding, Requests will use the new value of ``r.encoding`` whenever you call ``r.text``. You might want to do this in any situation where you can apply special logic to work out what the encoding of the content will -be. For example, HTTP and XML have the ability to specify their encoding in +be. For example, HTML and XML have the ability to specify their encoding in their body. In situations like this, you should use ``r.content`` to find the encoding, and then set ``r.encoding``. This will let you use ``r.text`` with the correct encoding. @@ -189,6 +189,14 @@ download, the above is the preferred and recommended way to retrieve the content. Note that ``chunk_size`` can be freely adjusted to a number that may better fit your use cases. +.. note:: + + An important note about using ``Response.iter_content`` versus ``Response.raw``. + ``Response.iter_content`` will automatically decode the ``gzip`` and ``deflate`` + transfer-encodings. ``Response.raw`` is a raw stream of bytes -- it does not + transform the response content. If you really need access to the bytes as they + were returned, use ``Response.raw``. + Custom Headers -------------- @@ -273,6 +281,7 @@ the ``json`` parameter (added in version 2.4.2) and it will be encoded automatic >>> r = requests.post(url, json=payload) +Note, the ``json`` parameter is ignored if either ``data`` or ``files`` is passed. POST a Multipart-Encoded File ----------------------------- diff --git a/requests/__init__.py b/requests/__init__.py index 268e7dcc..6fa855df 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -71,6 +71,17 @@ def check_compatibility(urllib3_version, chardet_version): assert patch >= 2 +def _check_cryptography(cryptography_version): + # cryptography < 1.3.4 + try: + cryptography_version = list(map(int, cryptography_version.split('.'))) + except ValueError: + return + + if cryptography_version < [1, 3, 4]: + warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version) + warnings.warn(warning, RequestsDependencyWarning) + # Check imported dependencies for compatibility. try: check_compatibility(urllib3.__version__, chardet.__version__) @@ -83,6 +94,10 @@ except (AssertionError, ValueError): try: from urllib3.contrib import pyopenssl pyopenssl.inject_into_urllib3() + + # Check cryptography version + from cryptography import __version__ as cryptography_version + _check_cryptography(cryptography_version) except ImportError: pass diff --git a/requests/adapters.py b/requests/adapters.py index 00f8792b..bc01e336 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -13,6 +13,7 @@ import socket from urllib3.poolmanager import PoolManager, proxy_from_url from urllib3.response import HTTPResponse +from urllib3.util import parse_url from urllib3.util import Timeout as TimeoutSauce from urllib3.util.retry import Retry from urllib3.exceptions import ClosedPoolError @@ -28,13 +29,13 @@ from urllib3.exceptions import ResponseError from .models import Response from .compat import urlparse, basestring -from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers, - prepend_scheme_if_needed, get_auth_from_url, urldefragauth, - select_proxy) +from .utils import (DEFAULT_CA_BUNDLE_PATH, extract_zipped_paths, + get_encoding_from_headers, prepend_scheme_if_needed, + get_auth_from_url, urldefragauth, select_proxy) from .structures import CaseInsensitiveDict from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidSchema) + ProxyError, RetryError, InvalidSchema, InvalidProxyURL) from .auth import _basic_auth_str try: @@ -219,7 +220,7 @@ class HTTPAdapter(BaseAdapter): cert_loc = verify if not cert_loc: - cert_loc = DEFAULT_CA_BUNDLE_PATH + cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) if not cert_loc or not os.path.exists(cert_loc): raise IOError("Could not find a suitable TLS CA certificate bundle, " @@ -300,6 +301,10 @@ class HTTPAdapter(BaseAdapter): if proxy: proxy = prepend_scheme_if_needed(proxy, 'http') + proxy_url = parse_url(proxy) + if not proxy_url.host: + raise InvalidProxyURL("Please check proxy URL. It is malformed" + " and could be missing the host.") proxy_manager = self.proxy_manager_for(proxy) conn = proxy_manager.connection_from_url(url) else: diff --git a/requests/exceptions.py b/requests/exceptions.py index be7eaed6..a80cad80 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -85,6 +85,10 @@ class InvalidHeader(RequestException, ValueError): """The header value provided was somehow invalid.""" +class InvalidProxyURL(InvalidURL): + """The proxy URL provided is invalid.""" + + class ChunkedEncodingError(RequestException): """The server declared chunked encoding but sent an invalid chunk.""" diff --git a/requests/models.py b/requests/models.py index 4041cac3..ce4e284e 100644 --- a/requests/models.py +++ b/requests/models.py @@ -686,11 +686,11 @@ class Response(object): @property def ok(self): - """Returns True if :attr:`status_code` is less than 400. + """Returns True if :attr:`status_code` is less than 400, False if not. This attribute checks if the status code of the response is between 400 and 600 to see if there was a client error or a server error. If - the status code, is between 200 and 400, this will return True. This + the status code is between 200 and 400, this will return True. This is **not** a check to see if the response code is ``200 OK``. """ try: diff --git a/requests/sessions.py b/requests/sessions.py index 6570e733..2cedaa8f 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -8,7 +8,7 @@ This module provides a Session object to manage and persist settings across requests (cookies, auth, proxies). """ import os -import platform +import sys import time from collections import Mapping from datetime import timedelta @@ -38,8 +38,8 @@ from .status_codes import codes from .models import REDIRECT_STATI # Preferred clock, based on which one is more accurate on a given system. -if platform.system() == 'Windows': - try: # Python 3.3+ +if sys.platform == 'win32': + try: # Python 3.4+ preferred_clock = time.perf_counter except AttributeError: # Earlier than Python 3. preferred_clock = time.clock @@ -696,7 +696,7 @@ class Session(SessionRedirectMixin): """ for (prefix, adapter) in self.adapters.items(): - if url.lower().startswith(prefix): + if url.lower().startswith(prefix.lower()): return adapter # Nothing matches :-/ diff --git a/requests/status_codes.py b/requests/status_codes.py index dee89190..96b86ddb 100644 --- a/requests/status_codes.py +++ b/requests/status_codes.py @@ -1,5 +1,22 @@ # -*- coding: utf-8 -*- +""" +The ``codes`` object defines a mapping from common names for HTTP statuses +to their numerical codes, accessible either as attributes or as dictionary +items. + +>>> requests.codes['temporary_redirect'] +307 +>>> requests.codes.teapot +418 +>>> requests.codes['\o/'] +200 + +Some codes have multiple names, and both upper- and lower-case versions of +the names are allowed. For example, ``codes.ok``, ``codes.OK``, and +``codes.okay`` all correspond to the HTTP status code 200. +""" + from .structures import LookupDict _codes = { @@ -84,8 +101,19 @@ _codes = { codes = LookupDict(name='status_codes') -for code, titles in _codes.items(): - for title in titles: - setattr(codes, title, code) - if not title.startswith(('\\', '/')): - setattr(codes, title.upper(), code) +def _init(): + for code, titles in _codes.items(): + for title in titles: + setattr(codes, title, code) + if not title.startswith(('\\', '/')): + setattr(codes, title.upper(), code) + + def doc(code): + names = ', '.join('``%s``' % n for n in _codes[code]) + return '* %d: %s' % (code, names) + + global __doc__ + __doc__ = (__doc__ + '\n' + + '\n'.join(doc(code) for code in sorted(_codes))) + +_init() diff --git a/requests/utils.py b/requests/utils.py index c52ce2d0..df18a0bf 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -8,17 +8,18 @@ This module provides utility functions that are used within Requests that are also useful for external consumption. """ -import cgi import codecs import collections import contextlib import io import os -import platform import re import socket import struct +import sys +import tempfile import warnings +import zipfile from .__version__ import __version__ from . import certs @@ -39,19 +40,25 @@ NETRC_FILES = ('.netrc', '_netrc') DEFAULT_CA_BUNDLE_PATH = certs.where() -if platform.system() == 'Windows': +if sys.platform == 'win32': # provide a proxy_bypass version on Windows without DNS lookups def proxy_bypass_registry(host): - if is_py3: - import winreg - else: - import _winreg as winreg + try: + if is_py3: + import winreg + else: + import _winreg as winreg + except ImportError: + return False + try: internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r'Software\Microsoft\Windows\CurrentVersion\Internet Settings') - proxyEnable = winreg.QueryValueEx(internetSettings, - 'ProxyEnable')[0] + # ProxyEnable could be REG_SZ or REG_DWORD, normalizing it + proxyEnable = int(winreg.QueryValueEx(internetSettings, + 'ProxyEnable')[0]) + # ProxyOverride is almost always a string proxyOverride = winreg.QueryValueEx(internetSettings, 'ProxyOverride')[0] except OSError: @@ -216,6 +223,38 @@ def guess_filename(obj): return os.path.basename(name) +def extract_zipped_paths(path): + """Replace nonexistant paths that look like they refer to a member of a zip + archive with the location of an extracted copy of the target, or else + just return the provided path unchanged. + """ + if os.path.exists(path): + # this is already a valid path, no need to do anything further + return path + + # find the first valid part of the provided path and treat that as a zip archive + # assume the rest of the path is the name of a member in the archive + archive, member = os.path.split(path) + while archive and not os.path.exists(archive): + archive, prefix = os.path.split(archive) + member = '/'.join([prefix, member]) + + if not zipfile.is_zipfile(archive): + return path + + zip_file = zipfile.ZipFile(archive) + if member not in zip_file.namelist(): + return path + + # we have a valid zip archive and a valid member of that archive + tmp = tempfile.gettempdir() + extracted_path = os.path.join(tmp, *member.split('/')) + if not os.path.exists(extracted_path): + extracted_path = zip_file.extract(member, path=tmp) + + return extracted_path + + def from_key_val_list(value): """Take an object and test to see if it can be represented as a dictionary. Unless it can not be represented as such, return an @@ -407,6 +446,31 @@ def get_encodings_from_content(content): xml_re.findall(content)) +def _parse_content_type_header(header): + """Returns content type and parameters from given header + + :param header: string + :return: tuple containing content type and dictionary of + parameters + """ + + tokens = header.split(';') + content_type, params = tokens[0].strip(), tokens[1:] + params_dict = {} + items_to_strip = "\"' " + + for param in params: + param = param.strip() + if param: + key, value = param, True + index_of_equals = param.find("=") + if index_of_equals != -1: + key = param[:index_of_equals].strip(items_to_strip) + value = param[index_of_equals + 1:].strip(items_to_strip) + params_dict[key] = value + return content_type, params_dict + + def get_encoding_from_headers(headers): """Returns encodings from given HTTP Header Dict. @@ -419,7 +483,7 @@ def get_encoding_from_headers(headers): if not content_type: return None - content_type, params = cgi.parse_header(content_type) + content_type, params = _parse_content_type_header(content_type) if 'charset' in params: return params['charset'].strip("'\"") @@ -639,28 +703,31 @@ def should_bypass_proxies(url, no_proxy): no_proxy_arg = no_proxy if no_proxy is None: no_proxy = get_proxy('no_proxy') - netloc = urlparse(url).netloc + parsed = urlparse(url) if no_proxy: # We need to check whether we match here. We need to see if we match - # the end of the netloc, both with and without the port. + # the end of the hostname, both with and without the port. no_proxy = ( host for host in no_proxy.replace(' ', '').split(',') if host ) - ip = netloc.split(':')[0] - if is_ipv4_address(ip): + if is_ipv4_address(parsed.hostname): for proxy_ip in no_proxy: if is_valid_cidr(proxy_ip): - if address_in_network(ip, proxy_ip): + if address_in_network(parsed.hostname, proxy_ip): return True - elif ip == proxy_ip: + elif parsed.hostname == proxy_ip: # If no_proxy ip was defined in plain IP notation instead of cidr notation & # matches the IP of the index return True else: + host_with_port = parsed.hostname + if parsed.port: + host_with_port += ':{0}'.format(parsed.port) + for host in no_proxy: - if netloc.endswith(host) or netloc.split(':')[0].endswith(host): + if parsed.hostname.endswith(host) or host_with_port.endswith(host): # The URL does match something in no_proxy, so we don't want # to apply the proxies on this URL. return True @@ -673,7 +740,7 @@ def should_bypass_proxies(url, no_proxy): # legitimate problems. with set_environ('no_proxy', no_proxy_arg): try: - bypass = proxy_bypass(netloc) + bypass = proxy_bypass(parsed.hostname) except (TypeError, socket.gaierror): bypass = False diff --git a/setup.cfg b/setup.cfg index 2a9acf13..ed8a958e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ [bdist_wheel] universal = 1 + +[metadata] +license_file = LICENSE diff --git a/setup.py b/setup.py index 7aa8d611..f32cca75 100755 --- a/setup.py +++ b/setup.py @@ -72,6 +72,7 @@ setup( package_data={'': ['LICENSE', 'NOTICE'], 'requests': ['*.pem']}, package_dir={'requests': 'requests'}, include_package_data=True, + python_requires=">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", install_requires=requires, license=about['__license__'], zip_safe=False, @@ -81,6 +82,7 @@ setup( 'Natural Language :: English', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', + 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', @@ -98,4 +100,3 @@ setup( 'socks:sys_platform == "win32" and (python_version == "2.7" or python_version == "2.6")': ['win_inet_pton'], }, ) - diff --git a/tests/test_requests.py b/tests/test_requests.py index 4d399518..e6a026f2 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -23,7 +23,7 @@ from requests.cookies import ( from requests.exceptions import ( ConnectionError, ConnectTimeout, InvalidSchema, InvalidURL, MissingSchema, ReadTimeout, Timeout, RetryError, TooManyRedirects, - ProxyError, InvalidHeader, UnrewindableBodyError, SSLError) + ProxyError, InvalidHeader, UnrewindableBodyError, SSLError, InvalidProxyURL) from requests.models import PreparedRequest from requests.structures import CaseInsensitiveDict from requests.sessions import SessionRedirectMixin @@ -526,6 +526,19 @@ class TestRequests: with pytest.raises(ProxyError): requests.get('http://localhost:1', proxies={'http': 'non-resolvable-address'}) + def test_proxy_error_on_bad_url(self, httpbin, httpbin_secure): + with pytest.raises(InvalidProxyURL): + requests.get(httpbin_secure(), proxies={'https': 'http:/badproxyurl:3128'}) + + with pytest.raises(InvalidProxyURL): + requests.get(httpbin(), proxies={'http': 'http://:8080'}) + + with pytest.raises(InvalidProxyURL): + requests.get(httpbin_secure(), proxies={'https': 'https://'}) + + with pytest.raises(InvalidProxyURL): + requests.get(httpbin(), proxies={'http': 'http:///example.com:8080'}) + def test_basicauth_with_netrc(self, httpbin): auth = ('user', 'pass') wrong_auth = ('wronguser', 'wrongpass') @@ -1351,6 +1364,44 @@ class TestRequests: assert 'http://' in s2.adapters assert 'https://' in s2.adapters + def test_session_get_adapter_prefix_matching(self, httpbin): + prefix = 'https://example.com' + more_specific_prefix = prefix + '/some/path' + + url_matching_only_prefix = prefix + '/another/path' + url_matching_more_specific_prefix = more_specific_prefix + '/longer/path' + url_not_matching_prefix = 'https://another.example.com/' + + s = requests.Session() + prefix_adapter = HTTPAdapter() + more_specific_prefix_adapter = HTTPAdapter() + s.mount(prefix, prefix_adapter) + s.mount(more_specific_prefix, more_specific_prefix_adapter) + + assert s.get_adapter(url_matching_only_prefix) is prefix_adapter + assert s.get_adapter(url_matching_more_specific_prefix) is more_specific_prefix_adapter + assert s.get_adapter(url_not_matching_prefix) not in (prefix_adapter, more_specific_prefix_adapter) + + def test_session_get_adapter_prefix_matching_mixed_case(self, httpbin): + mixed_case_prefix = 'hTtPs://eXamPle.CoM/MixEd_CAse_PREfix' + url_matching_prefix = mixed_case_prefix + '/full_url' + + s = requests.Session() + my_adapter = HTTPAdapter() + s.mount(mixed_case_prefix, my_adapter) + + assert s.get_adapter(url_matching_prefix) is my_adapter + + def test_session_get_adapter_prefix_matching_is_case_insensitive(self, httpbin): + mixed_case_prefix = 'hTtPs://eXamPle.CoM/MixEd_CAse_PREfix' + url_matching_prefix_with_different_case = 'HtTpS://exaMPLe.cOm/MiXeD_caSE_preFIX/another_url' + + s = requests.Session() + my_adapter = HTTPAdapter() + s.mount(mixed_case_prefix, my_adapter) + + assert s.get_adapter(url_matching_prefix_with_different_case) is my_adapter + def test_header_remove_is_case_insensitive(self, httpbin): # From issue #1321 s = requests.Session() diff --git a/tests/test_utils.py b/tests/test_utils.py index 32e4d4a5..f39cd67b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,15 +2,18 @@ import os import copy +import filecmp from io import BytesIO +import zipfile +from collections import deque import pytest from requests import compat from requests.cookies import RequestsCookieJar from requests.structures import CaseInsensitiveDict from requests.utils import ( - address_in_network, dotted_netmask, - get_auth_from_url, get_encoding_from_headers, + address_in_network, dotted_netmask, extract_zipped_paths, + get_auth_from_url, _parse_content_type_header, get_encoding_from_headers, get_encodings_from_content, get_environ_proxies, guess_filename, guess_json_utf, is_ipv4_address, is_valid_cidr, iter_slices, parse_dict_header, @@ -256,6 +259,32 @@ class TestGuessFilename: assert isinstance(result, expected_type) +class TestExtractZippedPaths: + + @pytest.mark.parametrize( + 'path', ( + '/', + __file__, + pytest.__file__, + '/etc/invalid/location', + )) + def test_unzipped_paths_unchanged(self, path): + assert path == extract_zipped_paths(path) + + def test_zipped_paths_extracted(self, tmpdir): + zipped_py = tmpdir.join('test.zip') + with zipfile.ZipFile(zipped_py.strpath, 'w') as f: + f.write(__file__) + + _, name = os.path.splitdrive(__file__) + zipped_path = os.path.join(zipped_py.strpath, name.lstrip(r'\/')) + extracted_path = extract_zipped_paths(zipped_path) + + assert extracted_path != zipped_path + assert os.path.exists(extracted_path) + assert filecmp.cmp(extracted_path, __file__) + + class TestContentEncodingDetection: def test_none(self): @@ -441,6 +470,45 @@ def test_parse_dict_header(value, expected): assert parse_dict_header(value) == expected +@pytest.mark.parametrize( + 'value, expected', ( + ( + 'application/xml', + ('application/xml', {}) + ), + ( + 'application/json ; charset=utf-8', + ('application/json', {'charset': 'utf-8'}) + ), + ( + 'text/plain', + ('text/plain', {}) + ), + ( + 'multipart/form-data; boundary = something ; boundary2=\'something_else\' ; no_equals ', + ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) + ), + ( + 'multipart/form-data; boundary = something ; boundary2="something_else" ; no_equals ', + ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) + ), + ( + 'multipart/form-data; boundary = something ; \'boundary2=something_else\' ; no_equals ', + ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) + ), + ( + 'multipart/form-data; boundary = something ; "boundary2=something_else" ; no_equals ', + ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) + ), + ( + 'application/json ; ; ', + ('application/json', {}) + ) + )) +def test__parse_content_type_header(value, expected): + assert _parse_content_type_header(value) == expected + + @pytest.mark.parametrize( 'value, expected', ( ( @@ -546,6 +614,7 @@ def test_urldefragauth(url, expected): ('http://172.16.1.1/', True), ('http://172.16.1.1:5000/', True), ('http://localhost.localdomain:5000/v1.0/', True), + ('http://google.com:6000/', True), ('http://172.16.1.12/', False), ('http://172.16.1.12:5000/', False), ('http://google.com:5000/v1.0/', False), @@ -554,11 +623,31 @@ def test_should_bypass_proxies(url, expected, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not """ - monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') - monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') + monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000') + monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000') assert should_bypass_proxies(url, no_proxy=None) == expected +@pytest.mark.parametrize( + 'url, expected', ( + ('http://172.16.1.1/', '172.16.1.1'), + ('http://172.16.1.1:5000/', '172.16.1.1'), + ('http://user:pass@172.16.1.1', '172.16.1.1'), + ('http://user:pass@172.16.1.1:5000', '172.16.1.1'), + ('http://hostname/', 'hostname'), + ('http://hostname:5000/', 'hostname'), + ('http://user:pass@hostname', 'hostname'), + ('http://user:pass@hostname:5000', 'hostname'), + )) +def test_should_bypass_proxies_pass_only_hostname(url, expected, mocker): + """The proxy_bypass function should be called with a hostname or IP without + a port number or auth credentials. + """ + proxy_bypass = mocker.patch('requests.utils.proxy_bypass') + should_bypass_proxies(url, no_proxy=None) + proxy_bypass.assert_called_once_with(expected) + + @pytest.mark.parametrize( 'cookiejar', ( compat.cookielib.CookieJar(), @@ -638,6 +727,7 @@ def test_should_bypass_proxies_win_registry(url, expected, override, pass ie_settings = RegHandle() + proxyEnableValues = deque([1, "1"]) def OpenKey(key, subkey): return ie_settings @@ -645,7 +735,9 @@ def test_should_bypass_proxies_win_registry(url, expected, override, def QueryValueEx(key, value_name): if key is ie_settings: if value_name == 'ProxyEnable': - return [1] + # this could be a string (REG_SZ) or a 32-bit number (REG_DWORD) + proxyEnableValues.rotate() + return [proxyEnableValues[0]] elif value_name == 'ProxyOverride': return [override] @@ -656,6 +748,7 @@ def test_should_bypass_proxies_win_registry(url, expected, override, monkeypatch.setenv('NO_PROXY', '') monkeypatch.setattr(winreg, 'OpenKey', OpenKey) monkeypatch.setattr(winreg, 'QueryValueEx', QueryValueEx) + assert should_bypass_proxies(url, None) == expected @pytest.mark.parametrize( diff --git a/tox.ini b/tox.ini index 2a961c82..38bf3ac4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] -envlist = py26,py27,py33,py34,py35,py36 +envlist = py26,py27,py34,py35,py36 [testenv] commands = pip install -e .[socks] - python setup.py test \ No newline at end of file + python setup.py test