From f123f89d32d3258429cdecf33fbd20e68a746042 Mon Sep 17 00:00:00 2001 From: Dmitry Dygalo Date: Fri, 19 Feb 2016 10:32:23 +0100 Subject: [PATCH] Added unit tests for utils module --- requests/utils.py | 25 +++--- tests/test_utils.py | 204 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 205 insertions(+), 24 deletions(-) diff --git a/requests/utils.py b/requests/utils.py index c5c3fd01..b4ceb1ef 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -14,9 +14,7 @@ import codecs import collections import io import os -import platform import re -import sys import socket import struct import warnings @@ -557,6 +555,7 @@ def should_bypass_proxies(url): return False + def get_environ_proxies(url): """Return a dict of environment proxies.""" if should_bypass_proxies(url): @@ -564,6 +563,7 @@ def get_environ_proxies(url): else: return getproxies() + def select_proxy(url, proxies): """Select a proxy for the url, if applicable. @@ -577,6 +577,7 @@ def select_proxy(url, proxies): proxy = proxies.get(urlparts.scheme) return proxy + def default_user_agent(name="python-requests"): """Return a string representing the default user agent.""" return '%s/%s' % (name, __version__) @@ -600,21 +601,19 @@ def parse_header_links(value): links = [] - replace_chars = " '\"" + replace_chars = ' \'"' - for val in re.split(", *<", value): + for val in re.split(', *<', value): try: - url, params = val.split(";", 1) + url, params = val.split(';', 1) except ValueError: url, params = val, '' - link = {} + link = {'url': url.strip('<> \'"')} - link["url"] = url.strip("<> '\"") - - for param in params.split(";"): + for param in params.split(';'): try: - key, value = param.split("=") + key, value = param.split('=') except ValueError: break @@ -661,8 +660,8 @@ def guess_json_utf(data): def prepend_scheme_if_needed(url, new_scheme): - '''Given a URL that may or may not have a scheme, prepend the given scheme. - Does not replace a present scheme with the one provided as an argument.''' + """Given a URL that may or may not have a scheme, prepend the given scheme. + Does not replace a present scheme with the one provided as an argument.""" scheme, netloc, path, params, query, fragment = urlparse(url, new_scheme) # urlparse is a finicky beast, and sometimes decides that there isn't a @@ -693,8 +692,6 @@ def to_native_string(string, encoding='ascii'): string in the native string type, encoding and decoding where necessary. This assumes ASCII unless told otherwise. """ - out = None - if isinstance(string, builtin_str): out = string else: diff --git a/tests/test_utils.py b/tests/test_utils.py index 5a50e366..3d9d3936 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,15 +1,20 @@ # coding: utf-8 -import os from io import BytesIO import pytest from requests import compat +from requests.structures import CaseInsensitiveDict from requests.utils import ( address_in_network, dotted_netmask, - get_auth_from_url, get_encodings_from_content, - get_environ_proxies, guess_filename, - is_ipv4_address, is_valid_cidr, requote_uri, - select_proxy, super_len) + get_auth_from_url, get_encoding_from_headers, + get_encodings_from_content, get_environ_proxies, + guess_filename, guess_json_utf, is_ipv4_address, + is_valid_cidr, iter_slices, parse_dict_header, + parse_header_links, prepend_scheme_if_needed, + requote_uri, select_proxy, super_len, + to_key_val_list, to_native_string, + unquote_header_value, unquote_unreserved, + urldefragauth) from .compat import StringIO, cStringIO @@ -33,16 +38,63 @@ class TestSuperLen: s.write('foobarbogus') assert super_len(s) == 0 + def test_string(self): + assert super_len('Test') == 4 + + @pytest.mark.parametrize( + 'mode, warnings_num', ( + ('r', 1), + ('rb', 0), + )) + def test_file(self, tmpdir, mode, warnings_num, recwarn): + file_obj = tmpdir.join('test.txt') + file_obj.write('Test') + with file_obj.open(mode) as fd: + assert super_len(fd) == 4 + assert len(recwarn) == warnings_num + + +class TestToKeyValList: + + @pytest.mark.parametrize( + 'value, expected', ( + ([('key', 'val')], [('key', 'val')]), + ((('key', 'val'), ), [('key', 'val')]), + ({'key': 'val'}, [('key', 'val')]), + (None, None) + )) + def test_valid(self, value, expected): + assert to_key_val_list(value) == expected + + def test_invalid(self): + with pytest.raises(ValueError): + to_key_val_list('string') + + +class TestUnquoteHeaderValue: + + @pytest.mark.parametrize( + 'value, expected', ( + (None, None), + ('Test', 'Test'), + ('"Test"', 'Test'), + ('"Test\\\\"', 'Test\\'), + ('"\\\\Comp\\Res"', '\\Comp\\Res'), + )) + def test_valid(self, value, expected): + assert unquote_header_value(value) == expected + + def test_is_filename(self): + assert unquote_header_value('"\\\\Comp\\Res"', True) == '\\\\Comp\\Res' + class TestGetEnvironProxies: """Ensures that IP addresses are correctly matches with ranges in no_proxy variable.""" - @pytest.yield_fixture(scope='class', autouse=True, params=['no_proxy', 'NO_PROXY']) - def no_proxy(self, request): - os.environ[request.param] = '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1' - yield - del os.environ[request.param] + @pytest.fixture(autouse=True, params=['no_proxy', 'NO_PROXY']) + def no_proxy(self, request, monkeypatch): + monkeypatch.setenv(request.param, '192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1') @pytest.mark.parametrize( 'url', ( @@ -152,6 +204,21 @@ class TestContentEncodingDetection: assert get_encodings_from_content(content) == ['HTML5', 'HTML4', 'XML'] +class TestGuessJSONUTF: + + @pytest.mark.parametrize( + 'encoding', ( + 'utf-32', 'utf-8-sig', 'utf-16', 'utf-8', 'utf-16-be', 'utf-16-le', + 'utf-32-be', 'utf-32-le' + )) + def test_encoded(self, encoding): + data = '{}'.encode(encoding) + assert guess_json_utf(data) == encoding + + def test_bad_utf_like_encoding(self): + assert guess_json_utf(b'\x00\x00\x00\x00') is None + + USER = PASSWORD = "%!*'();:@&=+$,/?#[] " ENCODED_USER = compat.quote(USER, '') ENCODED_PASSWORD = compat.quote(PASSWORD, '') @@ -184,6 +251,10 @@ ENCODED_PASSWORD = compat.quote(PASSWORD, '') 'http://user:pass%23pass@complex.url.com/path?query=yes', ('user', 'pass#pass') ), + ( + 'http://complex.url.com/path?query=yes', + ('', '') + ), )) def test_get_auth_from_url(url, auth): assert get_auth_from_url(url) == auth @@ -208,6 +279,23 @@ def test_requote_uri_with_unquoted_percents(uri, expected): assert requote_uri(uri) == expected +@pytest.mark.parametrize( + 'uri, expected', ( + ( + # Illegal bytes + 'http://example.com/?a=%--', + 'http://example.com/?a=%--', + ), + ( + # Reserved characters + 'http://example.com/?a=%300', + 'http://example.com/?a=00', + ) + )) +def test_unquote_unreserved(uri, expected): + assert unquote_unreserved(uri) == expected + + @pytest.mark.parametrize( 'mask, expected', ( (8, '255.0.0.0'), @@ -229,3 +317,99 @@ def test_select_proxies(url, expected): proxies = {'http': 'http://http.proxy', 'http://some.host': 'http://some.host.proxy'} assert select_proxy(url, proxies) == expected + + +@pytest.mark.parametrize( + 'value, expected', ( + ('foo="is a fish", bar="as well"', {'foo': 'is a fish', 'bar': 'as well'}), + ('key_without_value', {'key_without_value': None}) + )) +def test_parse_dict_header(value, expected): + assert parse_dict_header(value) == expected + + +@pytest.mark.parametrize( + 'value, expected', ( + ( + CaseInsensitiveDict(), + None + ), + ( + CaseInsensitiveDict({'content-type': 'application/json; charset=utf-8'}), + 'utf-8' + ), + ( + CaseInsensitiveDict({'content-type': 'text/plain'}), + 'ISO-8859-1' + ), + )) +def test_get_encoding_from_headers(value, expected): + assert get_encoding_from_headers(value) == expected + + +@pytest.mark.parametrize( + 'value, length', ( + ('', 0), + ('T', 1), + ('Test', 4), + )) +def test_iter_slices(value, length): + assert len(list(iter_slices(value, 1))) == length + + +@pytest.mark.parametrize( + 'value, expected', ( + ( + '; rel=front; type="image/jpeg"', + [{'url': 'http:/.../front.jpeg', 'rel': 'front', 'type': 'image/jpeg'}] + ), + ( + '', + [{'url': 'http:/.../front.jpeg'}] + ), + ( + ';', + [{'url': 'http:/.../front.jpeg'}] + ), + ( + '; type="image/jpeg",;', + [ + {'url': 'http:/.../front.jpeg', 'type': 'image/jpeg'}, + {'url': 'http://.../back.jpeg'} + ] + ), + )) +def test_parse_header_links(value, expected): + assert parse_header_links(value) == expected + + +@pytest.mark.parametrize( + 'value, expected', ( + ('example.com/path', 'http://example.com/path'), + ('//example.com/path', 'http://example.com/path'), + )) +def test_prepend_scheme_if_needed(value, expected): + assert prepend_scheme_if_needed(value, 'http') == expected + + +@pytest.mark.parametrize( + 'value, expected', ( + ('T', 'T'), + (b'T', 'T'), + (u'T', 'T'), + )) +def test_to_native_string(value, expected): + assert to_native_string(value) == expected + + +@pytest.mark.parametrize( + 'url, expected', ( + ('http://u:p@example.com/path?a=1#test', 'http://example.com/path?a=1'), + ('http://example.com/path', 'http://example.com/path'), + ('//u:p@example.com/path', '//example.com/path'), + ('//example.com/path', '//example.com/path'), + ('example.com/path', '//example.com/path'), + ('scheme:u:p@example.com/path', 'scheme://example.com/path'), + )) +def test_urldefragauth(url, expected): + assert urldefragauth(url) == expected