diff --git a/.travis.yml b/.travis.yml index b7693f63..aae4b560 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ python: - "3.4" - "3.5" - "3.6" - # - "3.7-dev" + - "3.7-dev" # - "pypy" -- appears to hang # - "pypy3" # command to install dependencies @@ -27,5 +27,3 @@ jobs: - stage: coverage python: 3.6 script: codecov - - diff --git a/AUTHORS.rst b/AUTHORS.rst index 8379f65c..2b4494ba 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -181,3 +181,5 @@ Patches and Suggestions - Taylor Hoff (`@PrimordialHelios `_) - Arthur Vigil (`@ahvigil `_) - Nehal J Wani (`@nehaljwani `_) +- Demetrios Bairaktaris (`@DemetriosBairaktaris `_) +- Darren Dormer (`@ddormer `_) diff --git a/HISTORY.rst b/HISTORY.rst index b099ecdb..61f7e232 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,7 @@ dev - Warn user about possible slowdown when using cryptography version < 1.3.4 - Check for invalid host in proxy URL, before forwarding request to adapter. +- Fragments are now properly maintained across redirects. (RFC7231 7.1.2) **Bugfixes** @@ -17,7 +18,8 @@ dev - Fixed issue where loading the default certificate bundle from a zip archive would raise an ``IOError`` - Fixed issue with unexpected ``ImportError`` on windows system which do not support ``winreg`` module - +- DNS resolution in proxy bypass no longer includes the username and password in + the request. This also fixes the issue of DNS queries failing on macOS. 2.18.4 (2017-08-15) +++++++++++++++++++ @@ -129,7 +131,7 @@ dev - Further restored the ``requests.packages`` namespace for compatibility reasons. -No code modification (noted below) should be neccessary any longer. +No code modification (noted below) should be necessary any longer. 2.16.1 (2017-05-27) +++++++++++++++++++ diff --git a/README.rst b/README.rst index 915f2a11..c7b033be 100644 --- a/README.rst +++ b/README.rst @@ -105,6 +105,6 @@ How to Contribute #. Write a test which shows that the bug was fixed or that the feature works as expected. #. Send a pull request and bug the maintainer until it gets merged and published. :) Make sure to add yourself to AUTHORS_. -.. _`the repository`: http://github.com/requests/requests +.. _`the repository`: https://github.com/requests/requests .. _AUTHORS: https://github.com/requests/requests/blob/master/AUTHORS.rst .. _Contributor Friendly: https://github.com/requests/requests/issues?direction=desc&labels=Contributor+Friendly&page=1&sort=updated&state=open diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html index 5b437d85..0b35068c 100644 --- a/docs/_templates/sidebarintro.html +++ b/docs/_templates/sidebarintro.html @@ -5,7 +5,7 @@

-

@@ -20,7 +20,7 @@

Stay Informed

Receive updates on new releases and upcoming projects.

-

@@ -30,7 +30,8 @@

More Kenneth Reitz projects:

@@ -67,4 +68,3 @@
  • Italian
  • Spanish
    • - diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index b31c3477..6453fa2b 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -4,7 +4,7 @@

    -

    @@ -25,7 +25,7 @@

    If you enjoy using this project, Say Thanks!

    -

    @@ -35,7 +35,8 @@

    More Kenneth Reitz projects:

    - diff --git a/docs/api.rst b/docs/api.rst index c3e00e54..ef84bf60 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -181,7 +181,7 @@ API Changes logging.basicConfig() # you need to initialize logging, otherwise you will not see anything from requests logging.getLogger().setLevel(logging.DEBUG) - requests_log = logging.getLogger("requests.packages.urllib3") + requests_log = logging.getLogger("urllib3") requests_log.setLevel(logging.DEBUG) requests_log.propagate = True diff --git a/docs/community/out-there.rst b/docs/community/out-there.rst index 5ce5f79f..63e70169 100644 --- a/docs/community/out-there.rst +++ b/docs/community/out-there.rst @@ -18,7 +18,7 @@ Articles & Talks - `Python for the Web `_ teaches how to use Python to interact with the web, using Requests. - `Daniel Greenfeld's Review of Requests `_ - `My 'Python for Humans' talk `_ ( `audio `_ ) -- `Issac Kelly's 'Consuming Web APIs' talk `_ +- `Issac Kelly's 'Consuming Web APIs' talk `_ - `Blog post about Requests via Yum `_ - `Russian blog post introducing Requests `_ - `Sending JSON in Requests `_ diff --git a/docs/community/recommended.rst b/docs/community/recommended.rst index 0f652d54..8fcd47a4 100644 --- a/docs/community/recommended.rst +++ b/docs/community/recommended.rst @@ -15,7 +15,7 @@ Certifi CA Bundle validating the trustworthiness of SSL certificates while verifying the identity of TLS hosts. It has been extracted from the Requests project. -.. _Certifi: http://certifi.io/en/latest/ +.. _Certifi: https://github.com/certifi/python-certifi CacheControl ------------ @@ -34,7 +34,7 @@ but do not belong in Requests proper. This library is actively maintained by members of the Requests core team, and reflects the functionality most requested by users within the community. -.. _Requests-Toolbelt: http://toolbelt.readthedocs.io/en/latest/index.html +.. _Requests-Toolbelt: https://toolbelt.readthedocs.io/en/latest/index.html Requests-Threads @@ -62,6 +62,3 @@ Betamax A VCR imitation designed only for Python-Requests. .. _betamax: https://github.com/sigmavirus24/betamax - - - diff --git a/docs/community/release-process.rst b/docs/community/release-process.rst index 2e317ceb..18f71168 100644 --- a/docs/community/release-process.rst +++ b/docs/community/release-process.rst @@ -19,19 +19,18 @@ Breaking changes are changes that break backwards compatibility with prior versions. If the project were to change the ``text`` attribute on a ``Response`` object to a method, that would only happen in a Major release. -Major releases may also include miscellaneous bug fixes and upgrades to -vendored packages. The core developers of Requests are committed to providing -a good user experience. This means we're also committed to preserving -backwards compatibility as much as possible. Major releases will be infrequent -and will need strong justifications before they are considered. +Major releases may also include miscellaneous bug fixes. The core developers of +Requests are committed to providing a good user experience. This means we're +also committed to preserving backwards compatibility as much as possible. Major +releases will be infrequent and will need strong justifications before they are +considered. Minor Releases -------------- -A minor release will not include breaking changes but may include -miscellaneous bug fixes and upgrades to vendored packages. If the previous -version of Requests released was ``v10.2.7`` a minor release would be -versioned as ``v10.3.0``. +A minor release will not include breaking changes but may include miscellaneous +bug fixes. If the previous version of Requests released was ``v10.2.7`` a minor +release would be versioned as ``v10.3.0``. Minor releases will be backwards compatible with releases that have the same major version number. In other words, all versions that would start with diff --git a/docs/community/updates.rst b/docs/community/updates.rst index f755a493..3b9a3097 100644 --- a/docs/community/updates.rst +++ b/docs/community/updates.rst @@ -29,4 +29,3 @@ Release and Version History =========================== .. include:: ../../HISTORY.rst - diff --git a/docs/conf.py b/docs/conf.py index 4bda98b0..c952fe79 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,7 +58,7 @@ master_doc = 'index' # General information about the project. project = u'Requests' -copyright = u'MMXVII. A Kenneth Reitz Project' +copyright = u'MMXVIII. A Kenneth Reitz Project' author = u'Kenneth Reitz' # The version info for the project you're documenting, acts as replacement for @@ -376,4 +376,4 @@ epub_exclude_files = ['search.html'] # If false, no index is generated. #epub_use_index = True -intersphinx_mapping = {'urllib3': ('http://urllib3.readthedocs.io/en/latest', None)} +intersphinx_mapping = {'urllib3': ('https://urllib3.readthedocs.io/en/latest', None)} diff --git a/docs/dev/todo.rst b/docs/dev/todo.rst index 50b18155..1766a28a 100644 --- a/docs/dev/todo.rst +++ b/docs/dev/todo.rst @@ -61,5 +61,4 @@ Requests currently supports the following versions of Python: Google AppEngine is not officially supported although support is available with the `Requests-Toolbelt`_. -.. _Requests-Toolbelt: http://toolbelt.readthedocs.io/ - +.. _Requests-Toolbelt: https://toolbelt.readthedocs.io/ diff --git a/docs/index.rst b/docs/index.rst index 72f93b90..7934a298 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,8 +30,8 @@ consumption. .. note:: The use of **Python 3** is *highly* preferred over Python 2. Consider upgrading your applications and infrastructure if you find yourself *still* using Python 2 in production today. If you are using Python 3, congratulations — you are indeed a person of excellent taste. —*Kenneth Reitz* - - + + ------------------- **Behold, the power of Requests**:: @@ -59,7 +59,7 @@ are 100% automatic, thanks to `urllib3 `_. User Testimonials ----------------- -Twitter, Spotify, Microsoft, Amazon, Lyft, BuzzFeed, Reddit, The NSA, Her Majesty's Government, Google, Twilio, Runscope, Mozilla, Heroku, +Nike, Twitter, Spotify, Microsoft, Amazon, Lyft, BuzzFeed, Reddit, The NSA, Her Majesty's Government, Google, Twilio, Runscope, Mozilla, Heroku, PayPal, NPR, Obama for America, Transifex, Native Instruments, The Washington Post, SoundCloud, Kippt, Sony, and Federal U.S. Institutions that prefer to be unnamed claim to use Requests internally. diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 587b3fdc..e5f7f297 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -196,18 +196,18 @@ As a result an ``SSL: CERTIFICATE_VERIFY_FAILED`` is thrown. You can get around this behaviour by explicity merging the environment settings into your session:: from requests import Request, Session - + s = Session() req = Request('GET', url) - + prepped = s.prepare_request(req) - + # Merge environment settings into session settings = s.merge_environment_settings(prepped.url, None, None, None, None) resp = s.send(prepped, **settings) - + print(resp.status_code) - + .. _verification: SSL Cert Verification @@ -274,20 +274,19 @@ If you specify a wrong path or an invalid cert, you'll get a SSLError:: CA Certificates --------------- -By default, Requests bundles a set of root CAs that it trusts, sourced from the -`Mozilla trust store`_. However, these are only updated once for each Requests -version. This means that if you pin a Requests version your certificates can -become extremely out of date. +Requests uses certificates from the package `certifi`_. This allows for users +to update their trusted certificates without changing the version of Requests. -From Requests version 2.4.0 onwards, Requests will attempt to use certificates -from `certifi`_ if it is present on the system. This allows for users to update -their trusted certificates without having to change the code that runs on their -system. +Before version 2.16, Requests bundled a set of root CAs that it trusted, +sourced from the `Mozilla trust store`_. The certificates were only updated +once for each Requests version. When ``certifi`` was not installed, this led to +extremely out-of-date certificate bundles when using significantly older +versions of Requests. For the sake of security we recommend upgrading certifi frequently! .. _HTTP persistent connection: https://en.wikipedia.org/wiki/HTTP_persistent_connection -.. _connection pooling: http://urllib3.readthedocs.io/en/latest/reference/index.html#module-urllib3.connectionpool +.. _connection pooling: https://urllib3.readthedocs.io/en/latest/reference/index.html#module-urllib3.connectionpool .. _certifi: http://certifi.io/ .. _Mozilla trust store: https://hg.mozilla.org/mozilla-central/raw-file/tip/security/nss/lib/ckfw/builtins/certdata.txt @@ -657,7 +656,7 @@ When you receive a response, Requests makes a guess at the encoding to use for decoding the response when you access the :attr:`Response.text ` attribute. Requests will first check for an encoding in the HTTP header, and if none is present, will use `chardet -`_ to attempt to guess the encoding. +`_ to attempt to guess the encoding. The only time Requests will not do this is if no explicit charset is present in the HTTP headers **and** the ``Content-Type`` @@ -884,7 +883,7 @@ Link Headers Many HTTP APIs feature Link headers. They make APIs more self describing and discoverable. -GitHub uses these for `pagination `_ +GitHub uses these for `pagination `_ in their API, for example:: >>> url = 'https://api.github.com/users/kennethreitz/repos?page=1&per_page=10' @@ -951,9 +950,9 @@ passed-through to `urllib3`. We'll make a Transport Adapter that instructs the library to use SSLv3:: import ssl + from urllib3.poolmanager import PoolManager from requests.adapters import HTTPAdapter - from requests.packages.urllib3.poolmanager import PoolManager class Ssl3HttpAdapter(HTTPAdapter): diff --git a/docs/user/authentication.rst b/docs/user/authentication.rst index 8ffab504..411f79fd 100644 --- a/docs/user/authentication.rst +++ b/docs/user/authentication.rst @@ -136,11 +136,11 @@ Further examples can be found under the `Requests organization`_ and in the .. _OAuth: http://oauth.net/ .. _requests_oauthlib: https://github.com/requests/requests-oauthlib -.. _requests-oauthlib OAuth2 documentation: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html -.. _Web Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#web-application-flow -.. _Mobile Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#mobile-application-flow -.. _Legacy Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#legacy-application-flow -.. _Backend Application Flow: http://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#backend-application-flow +.. _requests-oauthlib OAuth2 documentation: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html +.. _Web Application Flow: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#web-application-flow +.. _Mobile Application Flow: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#mobile-application-flow +.. _Legacy Application Flow: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#legacy-application-flow +.. _Backend Application Flow: https://requests-oauthlib.readthedocs.io/en/latest/oauth2_workflow.html#backend-application-flow .. _Kerberos: https://github.com/requests/requests-kerberos .. _NTLM: https://github.com/requests/requests-ntlm .. _Requests organization: https://github.com/requests diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 1a2c6fbf..b0ff231b 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -171,7 +171,7 @@ server, you can access ``r.raw``. If you want to do this, make sure you set >>> r = requests.get('https://api.github.com/events', stream=True) >>> r.raw - + >>> r.raw.read(10) '\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03' diff --git a/requests/adapters.py b/requests/adapters.py index bc01e336..a4b02842 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -411,7 +411,7 @@ class HTTPAdapter(BaseAdapter): self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) - self.add_headers(request) + self.add_headers(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) chunked = not (request.body is None or 'Content-Length' in request.headers) diff --git a/requests/help.py b/requests/help.py index 5440ee61..06e06b2a 100644 --- a/requests/help.py +++ b/requests/help.py @@ -13,7 +13,7 @@ import chardet from . import __version__ as requests_version try: - from .packages.urllib3.contrib import pyopenssl + from urllib3.contrib import pyopenssl except ImportError: pyopenssl = None OpenSSL = None diff --git a/requests/sessions.py b/requests/sessions.py index 2cedaa8f..4ffed46a 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -123,6 +123,7 @@ class SessionRedirectMixin(object): hist = [] # keep track of history url = self.get_redirect_target(resp) + previous_fragment = urlparse(req.url).fragment while url: prepared_request = req.copy() @@ -147,8 +148,12 @@ class SessionRedirectMixin(object): parsed_rurl = urlparse(resp.url) url = '%s:%s' % (to_native_string(parsed_rurl.scheme), url) - # The scheme should be lower case... + # Normalize url case and attach previous fragment if needed (RFC 7231 7.1.2) parsed = urlparse(url) + if parsed.fragment == '' and previous_fragment: + parsed = parsed._replace(fragment=previous_fragment) + elif parsed.fragment: + previous_fragment = parsed.fragment url = parsed.geturl() # Facilitate relative 'location' headers, as allowed by RFC 7231. diff --git a/requests/utils.py b/requests/utils.py index f9565287..3f50d485 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -8,7 +8,6 @@ This module provides utility functions that are used within Requests that are also useful for external consumption. """ -import cgi import codecs import collections import contextlib @@ -225,7 +224,7 @@ def guess_filename(obj): def extract_zipped_paths(path): - """Replace nonexistant paths that look like they refer to a member of a zip + """Replace nonexistent paths that look like they refer to a member of a zip archive with the location of an extracted copy of the target, or else just return the provided path unchanged. """ @@ -447,6 +446,31 @@ def get_encodings_from_content(content): xml_re.findall(content)) +def _parse_content_type_header(header): + """Returns content type and parameters from given header + + :param header: string + :return: tuple containing content type and dictionary of + parameters + """ + + tokens = header.split(';') + content_type, params = tokens[0].strip(), tokens[1:] + params_dict = {} + items_to_strip = "\"' " + + for param in params: + param = param.strip() + if param: + key, value = param, True + index_of_equals = param.find("=") + if index_of_equals != -1: + key = param[:index_of_equals].strip(items_to_strip) + value = param[index_of_equals + 1:].strip(items_to_strip) + params_dict[key] = value + return content_type, params_dict + + def get_encoding_from_headers(headers): """Returns encodings from given HTTP Header Dict. @@ -459,7 +483,7 @@ def get_encoding_from_headers(headers): if not content_type: return None - content_type, params = cgi.parse_header(content_type) + content_type, params = _parse_content_type_header(content_type) if 'charset' in params: return params['charset'].strip("'\"") @@ -679,28 +703,31 @@ def should_bypass_proxies(url, no_proxy): no_proxy_arg = no_proxy if no_proxy is None: no_proxy = get_proxy('no_proxy') - netloc = urlparse(url).netloc + parsed = urlparse(url) if no_proxy: # We need to check whether we match here. We need to see if we match - # the end of the netloc, both with and without the port. + # the end of the hostname, both with and without the port. no_proxy = ( host for host in no_proxy.replace(' ', '').split(',') if host ) - ip = netloc.split(':')[0] - if is_ipv4_address(ip): + if is_ipv4_address(parsed.hostname): for proxy_ip in no_proxy: if is_valid_cidr(proxy_ip): - if address_in_network(ip, proxy_ip): + if address_in_network(parsed.hostname, proxy_ip): return True - elif ip == proxy_ip: + elif parsed.hostname == proxy_ip: # If no_proxy ip was defined in plain IP notation instead of cidr notation & # matches the IP of the index return True else: + host_with_port = parsed.hostname + if parsed.port: + host_with_port += ':{0}'.format(parsed.port) + for host in no_proxy: - if netloc.endswith(host) or netloc.split(':')[0].endswith(host): + if parsed.hostname.endswith(host) or host_with_port.endswith(host): # The URL does match something in no_proxy, so we don't want # to apply the proxies on this URL. return True @@ -713,7 +740,7 @@ def should_bypass_proxies(url, no_proxy): # legitimate problems. with set_environ('no_proxy', no_proxy_arg): try: - bypass = proxy_bypass(netloc) + bypass = proxy_bypass(parsed.hostname) except (TypeError, socket.gaierror): bypass = False diff --git a/setup.py b/setup.py index e3bc8761..f32cca75 100755 --- a/setup.py +++ b/setup.py @@ -72,6 +72,7 @@ setup( package_data={'': ['LICENSE', 'NOTICE'], 'requests': ['*.pem']}, package_dir={'requests': 'requests'}, include_package_data=True, + python_requires=">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", install_requires=requires, license=about['__license__'], zip_safe=False, diff --git a/tests/test_lowlevel.py b/tests/test_lowlevel.py index c87234bc..6d6268cd 100644 --- a/tests/test_lowlevel.py +++ b/tests/test_lowlevel.py @@ -235,3 +235,75 @@ def test_redirect_rfc1808_to_non_ascii_location(): assert r.url == u'{0}/{1}'.format(url, expected_path.decode('ascii')) close_server.set() + +def test_fragment_not_sent_with_request(): + """Verify that the fragment portion of a URI isn't sent to the server.""" + def response_handler(sock): + req = consume_socket_content(sock, timeout=0.5) + sock.send( + b'HTTP/1.1 200 OK\r\n' + b'Content-Length: '+bytes(len(req))+b'\r\n' + b'\r\n'+req + ) + + close_server = threading.Event() + server = Server(response_handler, wait_to_close_event=close_server) + + with server as (host, port): + url = 'http://{0}:{1}/path/to/thing/#view=edit&token=hunter2'.format(host, port) + r = requests.get(url) + raw_request = r.content + + assert r.status_code == 200 + headers, body = raw_request.split(b'\r\n\r\n', 1) + status_line, headers = headers.split(b'\r\n', 1) + + assert status_line == b'GET /path/to/thing/ HTTP/1.1' + for frag in (b'view', b'edit', b'token', b'hunter2'): + assert frag not in headers + assert frag not in body + + close_server.set() + +def test_fragment_update_on_redirect(): + """Verify we only append previous fragment if one doesn't exist on new + location. If a new fragment is encounterd in a Location header, it should + be added to all subsequent requests. + """ + + def response_handler(sock): + consume_socket_content(sock, timeout=0.5) + sock.send( + b'HTTP/1.1 302 FOUND\r\n' + b'Content-Length: 0\r\n' + b'Location: /get#relevant-section\r\n\r\n' + ) + consume_socket_content(sock, timeout=0.5) + sock.send( + b'HTTP/1.1 302 FOUND\r\n' + b'Content-Length: 0\r\n' + b'Location: /final-url/\r\n\r\n' + ) + consume_socket_content(sock, timeout=0.5) + sock.send( + b'HTTP/1.1 200 OK\r\n\r\n' + ) + + close_server = threading.Event() + server = Server(response_handler, wait_to_close_event=close_server) + + with server as (host, port): + url = 'http://{0}:{1}/path/to/thing/#view=edit&token=hunter2'.format(host, port) + r = requests.get(url) + raw_request = r.content + + assert r.status_code == 200 + assert len(r.history) == 2 + assert r.history[0].request.url == url + + # Verify we haven't overwritten the location with our previous fragment. + assert r.history[1].request.url == 'http://{0}:{1}/get#relevant-section'.format(host, port) + # Verify previous fragment is used and not the original. + assert r.url == 'http://{0}:{1}/final-url/#relevant-section'.format(host, port) + + close_server.set() diff --git a/tests/test_requests.py b/tests/test_requests.py index fd85f7dc..0c7988dc 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -296,6 +296,14 @@ class TestRequests: for header in purged_headers: assert header not in next_resp.request.headers + def test_fragment_maintained_on_redirect(self, httpbin): + fragment = "#view=edit&token=hunter2" + r = requests.get(httpbin('redirect-to?url=get')+fragment) + + assert len(r.history) > 0 + assert r.history[0].request.url == httpbin('redirect-to?url=get')+fragment + assert r.url == httpbin('get')+fragment + def test_HTTP_200_OK_GET_WITH_PARAMS(self, httpbin): heads = {'User-agent': 'Mozilla/5.0'} diff --git a/tests/test_utils.py b/tests/test_utils.py index 2dd16923..f39cd67b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -13,7 +13,7 @@ from requests.cookies import RequestsCookieJar from requests.structures import CaseInsensitiveDict from requests.utils import ( address_in_network, dotted_netmask, extract_zipped_paths, - get_auth_from_url, get_encoding_from_headers, + get_auth_from_url, _parse_content_type_header, get_encoding_from_headers, get_encodings_from_content, get_environ_proxies, guess_filename, guess_json_utf, is_ipv4_address, is_valid_cidr, iter_slices, parse_dict_header, @@ -470,6 +470,45 @@ def test_parse_dict_header(value, expected): assert parse_dict_header(value) == expected +@pytest.mark.parametrize( + 'value, expected', ( + ( + 'application/xml', + ('application/xml', {}) + ), + ( + 'application/json ; charset=utf-8', + ('application/json', {'charset': 'utf-8'}) + ), + ( + 'text/plain', + ('text/plain', {}) + ), + ( + 'multipart/form-data; boundary = something ; boundary2=\'something_else\' ; no_equals ', + ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) + ), + ( + 'multipart/form-data; boundary = something ; boundary2="something_else" ; no_equals ', + ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) + ), + ( + 'multipart/form-data; boundary = something ; \'boundary2=something_else\' ; no_equals ', + ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) + ), + ( + 'multipart/form-data; boundary = something ; "boundary2=something_else" ; no_equals ', + ('multipart/form-data', {'boundary': 'something', 'boundary2': 'something_else', 'no_equals': True}) + ), + ( + 'application/json ; ; ', + ('application/json', {}) + ) + )) +def test__parse_content_type_header(value, expected): + assert _parse_content_type_header(value) == expected + + @pytest.mark.parametrize( 'value, expected', ( ( @@ -575,6 +614,7 @@ def test_urldefragauth(url, expected): ('http://172.16.1.1/', True), ('http://172.16.1.1:5000/', True), ('http://localhost.localdomain:5000/v1.0/', True), + ('http://google.com:6000/', True), ('http://172.16.1.12/', False), ('http://172.16.1.12:5000/', False), ('http://google.com:5000/v1.0/', False), @@ -583,11 +623,31 @@ def test_should_bypass_proxies(url, expected, monkeypatch): """Tests for function should_bypass_proxies to check if proxy can be bypassed or not """ - monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') - monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') + monkeypatch.setenv('no_proxy', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000') + monkeypatch.setenv('NO_PROXY', '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000') assert should_bypass_proxies(url, no_proxy=None) == expected +@pytest.mark.parametrize( + 'url, expected', ( + ('http://172.16.1.1/', '172.16.1.1'), + ('http://172.16.1.1:5000/', '172.16.1.1'), + ('http://user:pass@172.16.1.1', '172.16.1.1'), + ('http://user:pass@172.16.1.1:5000', '172.16.1.1'), + ('http://hostname/', 'hostname'), + ('http://hostname:5000/', 'hostname'), + ('http://user:pass@hostname', 'hostname'), + ('http://user:pass@hostname:5000', 'hostname'), + )) +def test_should_bypass_proxies_pass_only_hostname(url, expected, mocker): + """The proxy_bypass function should be called with a hostname or IP without + a port number or auth credentials. + """ + proxy_bypass = mocker.patch('requests.utils.proxy_bypass') + should_bypass_proxies(url, no_proxy=None) + proxy_bypass.assert_called_once_with(expected) + + @pytest.mark.parametrize( 'cookiejar', ( compat.cookielib.CookieJar(), diff --git a/tox.ini b/tox.ini index 2a961c82..38bf3ac4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] -envlist = py26,py27,py33,py34,py35,py36 +envlist = py26,py27,py34,py35,py36 [testenv] commands = pip install -e .[socks] - python setup.py test \ No newline at end of file + python setup.py test