diff --git a/AUTHORS.rst b/AUTHORS.rst index 1ff53148..f21d1736 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -110,3 +110,4 @@ Patches and Suggestions - Victoria Mo - Leila Muhtasib - Matthias Rahlf +- Jakub Roztocil diff --git a/docs/MANIFEST.in b/docs/MANIFEST.in index 403c87a6..fb1021bf 100644 --- a/docs/MANIFEST.in +++ b/docs/MANIFEST.in @@ -1 +1 @@ -include HISTORY.rst README.rst LICENSE \ No newline at end of file +include HISTORY.rst README.rst LICENSE \ No newline at end of file diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html index cb58776d..4d341223 100644 --- a/docs/_templates/sidebarintro.html +++ b/docs/_templates/sidebarintro.html @@ -5,7 +5,7 @@

-

diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index df79ff24..6a75a67d 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -4,7 +4,7 @@

-

diff --git a/docs/_themes/LICENSE b/docs/_themes/LICENSE index b160a8ee..3d1e04a2 100644 --- a/docs/_themes/LICENSE +++ b/docs/_themes/LICENSE @@ -1,9 +1,9 @@ -Modifications: +Modifications: Copyright (c) 2011 Kenneth Reitz. -Original Project: +Original Project: Copyright (c) 2010 by Armin Ronacher. diff --git a/docs/_themes/README.rst b/docs/_themes/README.rst index 2e875d46..de8310a2 100644 --- a/docs/_themes/README.rst +++ b/docs/_themes/README.rst @@ -1,7 +1,7 @@ krTheme Sphinx Style ==================== -This repository contains sphinx styles Kenneth Reitz uses in most of +This repository contains sphinx styles Kenneth Reitz uses in most of his projects. It is a derivative of Mitsuhiko's themes for Flask and Flask related projects. To use this style in your Sphinx documentation, follow this guide: diff --git a/docs/_themes/kr/theme.conf b/docs/_themes/kr/theme.conf index 307a1f0d..07698f6f 100644 --- a/docs/_themes/kr/theme.conf +++ b/docs/_themes/kr/theme.conf @@ -4,4 +4,4 @@ stylesheet = flasky.css pygments_style = flask_theme_support.FlaskyStyle [options] -touch_icon = +touch_icon = diff --git a/docs/_themes/kr_small/static/flasky.css_t b/docs/_themes/kr_small/static/flasky.css_t index fe2141c5..71961a27 100644 --- a/docs/_themes/kr_small/static/flasky.css_t +++ b/docs/_themes/kr_small/static/flasky.css_t @@ -8,11 +8,11 @@ * :license: BSD, see LICENSE for details. * */ - + @import url("basic.css"); - + /* -- page layout ----------------------------------------------------------- */ - + body { font-family: 'Georgia', serif; font-size: 17px; @@ -35,7 +35,7 @@ div.bodywrapper { hr { border: 1px solid #B1B4B6; } - + div.body { background-color: #ffffff; color: #3E4349; @@ -46,7 +46,7 @@ img.floatingflask { padding: 0 0 10px 10px; float: right; } - + div.footer { text-align: right; color: #888; @@ -55,12 +55,12 @@ div.footer { width: 650px; margin: 0 auto 40px auto; } - + div.footer a { color: #888; text-decoration: underline; } - + div.related { line-height: 32px; color: #888; @@ -69,18 +69,18 @@ div.related { div.related ul { padding: 0 0 0 10px; } - + div.related a { color: #444; } - + /* -- body styles ----------------------------------------------------------- */ - + a { color: #004B6B; text-decoration: underline; } - + a:hover { color: #6D4100; text-decoration: underline; @@ -89,7 +89,7 @@ a:hover { div.body { padding-bottom: 40px; /* saved for footer */ } - + div.body h1, div.body h2, div.body h3, @@ -109,24 +109,24 @@ div.indexwrapper h1 { height: {{ theme_index_logo_height }}; } {% endif %} - + div.body h2 { font-size: 180%; } div.body h3 { font-size: 150%; } div.body h4 { font-size: 130%; } div.body h5 { font-size: 100%; } div.body h6 { font-size: 100%; } - + a.headerlink { color: white; padding: 0 4px; text-decoration: none; } - + a.headerlink:hover { color: #444; background: #eaeaea; } - + div.body p, div.body dd, div.body li { line-height: 1.4em; } @@ -164,25 +164,25 @@ div.note { background-color: #eee; border: 1px solid #ccc; } - + div.seealso { background-color: #ffc; border: 1px solid #ff6; } - + div.topic { background-color: #eee; } - + div.warning { background-color: #ffe4e4; border: 1px solid #f66; } - + p.admonition-title { display: inline; } - + p.admonition-title:after { content: ":"; } @@ -254,7 +254,7 @@ dl { dl dd { margin-left: 30px; } - + pre { padding: 0; margin: 15px -30px; diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index adda9c73..0ac450d7 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -343,6 +343,31 @@ To use HTTP Basic Auth with your proxy, use the `http://user:password@host/` syn "http": "http://user:pass@10.10.1.10:3128/", } +Compliance +---------- + +Requests is intended to be compliant with all relevant specifications and +RFCs where that compliance will not cause difficulties for users. This +attention to the specification can lead to some behaviour that may seem +unusual to those not familiar with the relevant specification. + +Encodings +^^^^^^^^^ + +When you receive a response, Requests makes a guess at the encoding to use for +decoding the response when you call the ``Response.text`` method. Requests +will first check for an encoding in the HTTP header, and if none is present, +will use `chardet `_ to attempt to guess +the encoding. + +The only time Requests will not do this is if no explicit charset is present +in the HTTP headers **and** the ``Content-Type`` header contains ``text``. In +this situation, +`RFC 2616 `_ +specifies that the default charset must be ``ISO-8859-1``. Requests follows +the specification in this case. If you require a different encoding, you can +manually set the ``Response.encoding`` property, or use the raw +``Request.content``. HTTP Verbs ---------- diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 9b0399d4..c251e04b 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -24,7 +24,7 @@ Make a Request Making a request with Requests is very simple. Begin by importing the Requests module:: - + >>> import requests Now, let's try to get a webpage. For this example, let's get GitHub's public @@ -37,12 +37,12 @@ information we need from this object. Requests' simple API means that all forms of HTTP request are as obvious. For example, this is how you make an HTTP POST request:: - + >>> r = requests.post("http://httpbin.org/post") Nice, right? What about the other HTTP request types: PUT, DELETE, HEAD and OPTIONS? These are all just as simple:: - + >>> r = requests.put("http://httpbin.org/put") >>> r = requests.delete("http://httpbin.org/delete") >>> r = requests.head("http://httpbin.org/get") @@ -70,7 +70,7 @@ You can see that the URL has been correctly encoded by printing the URL:: >>> print r.url u'http://httpbin.org/get?key2=value2&key1=value1' - + Response Content ---------------- @@ -86,12 +86,22 @@ again:: Requests will automatically decode content from the server. Most unicode charsets are seamlessly decoded. -When you make a request, ``r.encoding`` is set, based on the HTTP headers. -Requests will use that encoding when you access ``r.text``. If ``r.encoding`` -is ``None``, Requests will make an extremely educated guess of the encoding -of the response body. You can manually set ``r.encoding`` to any encoding -you'd like, and that charset will be used. +When you make a request, Requests makes educated guesses about the encoding of +the response based on the HTTP headers. The text encoding guessed by Requests +is used when you access ``r.text``. You can find out what encoding Requests is +using, and change it, using the ``r.encoding`` property:: + >>> r.encoding + 'utf-8' + >>> r.encoding = 'ISO-8859-1' + +If you change the encoding, Requests will use the new value of ``r.encoding`` +whenever you call ``r.text``. + +Requests will also use custom encodings in the event that you need them. If +you have created your own encoding and registered it with the ``codecs`` +module, you can simply use the codec name as the value of ``r.encoding`` and +Requests will handle the decoding for you. Binary Response Content ----------------------- @@ -219,7 +229,7 @@ You can set the filename explicitly:: If you want, you can send strings to be received as files:: >>> url = 'http://httpbin.org/post' - >>> files = {'file': ('report.csv', 'some,data,to,send\nanother,row,to,send\n')} + >>> files = {'file': ('report.csv', 'some,data,to,send\nanother,row,to,send\n')} >>> r = requests.post(url, files=files) >>> r.text @@ -250,11 +260,11 @@ reference:: If we made a bad request (non-200 response), we can raise it with :class:`Response.raise_for_status()`:: - >>> _r = requests.get('http://httpbin.org/status/404') - >>> _r.status_code + >>> bad_r = requests.get('http://httpbin.org/status/404') + >>> bad_r.status_code 404 - >>> _r.raise_for_status() + >>> bad_r.raise_for_status() Traceback (most recent call last): File "requests/models.py", line 832, in raise_for_status raise http_error @@ -329,7 +339,7 @@ parameter:: Basic Authentication -------------------- -Many web services require authentication. There many different types of +Many web services require authentication. There are many different types of authentication, but the most common is HTTP Basic Auth. Making requests with Basic Auth is extremely simple:: diff --git a/requests/api.py b/requests/api.py index f192b8f5..ded79352 100644 --- a/requests/api.py +++ b/requests/api.py @@ -14,6 +14,7 @@ This module implements the Requests API. from . import sessions from .safe_mode import catch_exceptions_if_in_safe_mode + @catch_exceptions_if_in_safe_mode def request(method, url, **kwargs): """Constructs and sends a :class:`Request `. @@ -52,6 +53,7 @@ def request(method, url, **kwargs): if adhoc_session: session.close() + def get(url, **kwargs): """Sends a GET request. Returns :class:`Response` object. diff --git a/requests/auth.py b/requests/auth.py index 6aee69b2..38dd8741 100644 --- a/requests/auth.py +++ b/requests/auth.py @@ -34,6 +34,7 @@ log = logging.getLogger(__name__) CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded' + def _basic_auth_str(username, password): """Returns a Basic Auth string.""" @@ -239,6 +240,7 @@ class HTTPDigestAuth(AuthBase): r.register_hook('response', self.handle_401) return r + def _negotiate_value(r): """Extracts the gssapi authentication token from the appropriate header""" @@ -252,6 +254,7 @@ def _negotiate_value(r): return None + class HTTPKerberosAuth(AuthBase): """Attaches HTTP GSSAPI/Kerberos Authentication to the given Request object.""" def __init__(self, require_mutual_auth=True): diff --git a/requests/compat.py b/requests/compat.py index 201da3a9..d7012033 100644 --- a/requests/compat.py +++ b/requests/compat.py @@ -112,4 +112,3 @@ elif is_py3: bytes = bytes basestring = (str,bytes) numeric_types = (int, float) - diff --git a/requests/cookies.py b/requests/cookies.py index 04158561..bd2d6654 100644 --- a/requests/cookies.py +++ b/requests/cookies.py @@ -14,6 +14,7 @@ try: except ImportError: import dummy_threading as threading + class MockRequest(object): """Wraps a `requests.Request` to mimic a `urllib2.Request`. @@ -66,6 +67,7 @@ class MockRequest(object): def get_new_headers(self): return self._new_headers + class MockResponse(object): """Wraps a `httplib.HTTPMessage` to mimic a `urllib.addinfourl`. @@ -86,6 +88,7 @@ class MockResponse(object): def getheaders(self, name): self._headers.getheaders(name) + def extract_cookies_to_jar(jar, request, response): """Extract the cookies from the response into a CookieJar. @@ -99,12 +102,14 @@ def extract_cookies_to_jar(jar, request, response): res = MockResponse(response._original_response.msg) jar.extract_cookies(res, req) + def get_cookie_header(jar, request): """Produce an appropriate Cookie header string to be sent with `request`, or None.""" r = MockRequest(request) jar.add_cookie_header(r) return r.get_new_headers().get('Cookie') + def remove_cookie_by_name(cookiejar, name, domain=None, path=None): """Unsets a cookie by name, by default over all domains and paths. @@ -120,10 +125,12 @@ def remove_cookie_by_name(cookiejar, name, domain=None, path=None): for domain, path, name in clearables: cookiejar.clear(domain, path, name) + class CookieConflictError(RuntimeError): - """There are two cookies that meet the criteria specified in the cookie jar. + """There are two cookies that meet the criteria specified in the cookie jar. Use .get and .set and include domain and path args in order to be more specific.""" + class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): """Compatibility class; is a cookielib.CookieJar, but exposes a dict interface. @@ -181,7 +188,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): for cookie in iter(self): values.append(cookie.value) return values - + def items(self): """Dict-like items() that returns a list of name-value tuples from the jar. See keys() and values(). Allows client-code to call "dict(RequestsCookieJar) @@ -215,14 +222,14 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): if cookie.domain is not None and cookie.domain in domains: return True domains.append(cookie.domain) - return False # there is only one domain in jar + return False # there is only one domain in jar def get_dict(self, domain=None, path=None): """Takes as an argument an optional domain and path and returns a plain old Python dict of name-value pairs of cookies that meet the requirements.""" dictionary = {} for cookie in iter(self): - if (domain == None or cookie.domain == domain) and (path == None + if (domain == None or cookie.domain == domain) and (path == None or cookie.path == path): dictionary[cookie.name] = cookie.value return dictionary @@ -244,7 +251,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): remove_cookie_by_name(self, name) def _find(self, name, domain=None, path=None): - """Requests uses this method internally to get cookie values. Takes as args name + """Requests uses this method internally to get cookie values. Takes as args name and optional domain and path. Returns a cookie.value. If there are conflicting cookies, _find arbitrarily chooses one. See _find_no_duplicates if you want an exception thrown if there are conflicting cookies.""" @@ -257,18 +264,18 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): raise KeyError('name=%r, domain=%r, path=%r' % (name, domain, path)) def _find_no_duplicates(self, name, domain=None, path=None): - """__get_item__ and get call _find_no_duplicates -- never used in Requests internally. - Takes as args name and optional domain and path. Returns a cookie.value. - Throws KeyError if cookie is not found and CookieConflictError if there are + """__get_item__ and get call _find_no_duplicates -- never used in Requests internally. + Takes as args name and optional domain and path. Returns a cookie.value. + Throws KeyError if cookie is not found and CookieConflictError if there are multiple cookies that match name and optionally domain and path.""" toReturn = None for cookie in iter(self): if cookie.name == name: if domain is None or cookie.domain == domain: if path is None or cookie.path == path: - if toReturn != None: # if there are multiple cookies that meet passed in criteria + if toReturn != None: # if there are multiple cookies that meet passed in criteria raise CookieConflictError('There are multiple cookies with name, %r' % (name)) - toReturn = cookie.value # we will eventually return this as long as no cookie conflict + toReturn = cookie.value # we will eventually return this as long as no cookie conflict if toReturn: return toReturn @@ -291,6 +298,7 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): """This is not implemented. Calling this will throw an exception.""" raise NotImplementedError + def create_cookie(name, value, **kwargs): """Make a cookie from underspecified parameters. @@ -326,6 +334,7 @@ def create_cookie(name, value, **kwargs): return cookielib.Cookie(**result) + def morsel_to_cookie(morsel): """Convert a Morsel object into a Cookie containing the one k/v pair.""" c = create_cookie( @@ -349,6 +358,7 @@ def morsel_to_cookie(morsel): ) return c + def cookiejar_from_dict(cookie_dict, cookiejar=None): """Returns a CookieJar from a key/value dictionary. diff --git a/requests/exceptions.py b/requests/exceptions.py index 57f7b82d..6759af56 100644 --- a/requests/exceptions.py +++ b/requests/exceptions.py @@ -8,34 +8,44 @@ This module contains the set of Requests' exceptions. """ + class RequestException(RuntimeError): """There was an ambiguous exception that occurred while handling your request.""" + class HTTPError(RequestException): """An HTTP error occurred.""" response = None + class ConnectionError(RequestException): """A Connection error occurred.""" + class SSLError(ConnectionError): """An SSL error occurred.""" + class Timeout(RequestException): """The request timed out.""" + class URLRequired(RequestException): """A valid URL is required to make a request.""" + class TooManyRedirects(RequestException): """Too many redirects.""" + class MissingSchema(RequestException, ValueError): """The URL schema (e.g. http or https) is missing.""" + class InvalidSchema(RequestException, ValueError): """See defaults.py for valid schemas.""" + class InvalidURL(RequestException, ValueError): """ The URL provided was somehow invalid. """ diff --git a/requests/hooks.py b/requests/hooks.py index 272abb73..55bd9ac6 100644 --- a/requests/hooks.py +++ b/requests/hooks.py @@ -30,6 +30,7 @@ import traceback HOOKS = ('args', 'pre_request', 'pre_send', 'post_request', 'response') + def dispatch_hook(key, hooks, hook_data): """Dispatches a hook dictionary on a given piece of data.""" diff --git a/requests/models.py b/requests/models.py index 6a932543..f35ef7e1 100644 --- a/requests/models.py +++ b/requests/models.py @@ -8,7 +8,9 @@ This module contains the primary objects that power Requests. """ import os +import socket from datetime import datetime +from io import BytesIO from .hooks import dispatch_hook, HOOKS from .structures import CaseInsensitiveDict @@ -72,7 +74,14 @@ class Request(object): self.timeout = timeout #: Request URL. - self.url = url + #: Accept objects that have string representations. + try: + self.url = unicode(url) + except NameError: + # We're on Python 3. + self.url = str(url) + except UnicodeDecodeError: + self.url = url #: Dictionary of HTTP Headers to attach to the :class:`Request `. self.headers = dict(headers or []) @@ -292,7 +301,8 @@ class Request(object): proxies=self.proxies, verify=self.verify, session=self.session, - cert=self.cert + cert=self.cert, + prefetch=self.prefetch, ) request.send() @@ -328,7 +338,13 @@ class Request(object): return data def _encode_files(self, files): + """Build the body for a multipart/form-data request. + Will successfully encode files when passed as a dict or a list of + 2-tuples. Order is retained if data is a list of 2-tuples but abritrary + if parameters are supplied as a dict. + + """ if (not files) or isinstance(self.data, str): return None @@ -342,23 +358,22 @@ class Request(object): else: fn = guess_filename(v) or k fp = v - if isinstance(fp, (bytes, str)): + if isinstance(fp, str): fp = StringIO(fp) + if isinstance(fp, bytes): + fp = BytesIO(fp) fields.append((k, (fn, fp.read()))) new_fields = [] for field, val in fields: - if isinstance(val, float): - new_fields.append((field, str(val))) - elif isinstance(val, list): - newvalue = ', '.join(val) - new_fields.append((field, newvalue)) + if isinstance(val, list): + for v in val: + new_fields.append((k, str(v))) else: - new_fields.append((field, val)) - fields = new_fields - (body, content_type) = encode_multipart_formdata(fields) + new_fields.append((field, str(val))) + body, content_type = encode_multipart_formdata(new_fields) - return (body, content_type) + return body, content_type @property def full_url(self): @@ -378,7 +393,10 @@ class Request(object): if not scheme in SCHEMAS: raise InvalidSchema("Invalid scheme %r" % scheme) - netloc = netloc.encode('idna').decode('utf-8') + try: + netloc = netloc.encode('idna').decode('utf-8') + except UnicodeError: + raise InvalidURL('URL has an invalid label.') if not path: path = '/' @@ -452,7 +470,7 @@ class Request(object): except ValueError: return False - def send(self, anyway=False, prefetch=True): + def send(self, anyway=False, prefetch=None): """Sends the request. Returns True if successful, False if not. If there was an HTTPError during transmission, self.response.status_code will contain the HTTPError code. @@ -461,6 +479,9 @@ class Request(object): :param anyway: If True, request will be sent, even if it has already been sent. + + :param prefetch: If not None, will override the request's own setting + for prefetch. """ # Build the URL @@ -512,7 +533,7 @@ class Request(object): self.__dict__.update(r.__dict__) _p = urlparse(url) - no_proxy = filter(lambda x:x.strip(), self.proxies.get('no', '').split(',')) + no_proxy = filter(lambda x: x.strip(), self.proxies.get('no', '').split(',')) proxy = self.proxies.get(_p.scheme) if proxy and not any(map(_p.netloc.endswith, no_proxy)): @@ -598,6 +619,9 @@ class Request(object): ) self.sent = True + except socket.error as sockerr: + raise ConnectionError(sockerr) + except MaxRetryError as e: raise ConnectionError(e) @@ -620,7 +644,9 @@ class Request(object): self.__dict__.update(r.__dict__) # If prefetch is True, mark content as consumed. - if prefetch or self.prefetch: + if prefetch is None: + prefetch = self.prefetch + if prefetch: # Save the response. self.response.content diff --git a/requests/safe_mode.py b/requests/safe_mode.py index cd171f7d..0fb8d705 100644 --- a/requests/safe_mode.py +++ b/requests/safe_mode.py @@ -16,15 +16,16 @@ from .packages.urllib3.response import HTTPResponse from .exceptions import RequestException, ConnectionError, HTTPError import socket + def catch_exceptions_if_in_safe_mode(function): """New implementation of safe_mode. We catch all exceptions at the API level and then return a blank Response object with the error field filled. This decorator wraps request() in api.py. """ - + def wrapped(method, url, **kwargs): # if save_mode, we catch exceptions and fill error field - if (kwargs.get('config') and kwargs.get('config').get('safe_mode')) or (kwargs.get('session') + if (kwargs.get('config') and kwargs.get('config').get('safe_mode')) or (kwargs.get('session') and kwargs.get('session').config.get('safe_mode')): try: return function(method, url, **kwargs) @@ -32,8 +33,8 @@ def catch_exceptions_if_in_safe_mode(function): socket.timeout, socket.gaierror) as e: r = Response() r.error = e - r.raw = HTTPResponse() # otherwise, tests fail - r.status_code = 0 # with this status_code, content returns None + r.raw = HTTPResponse() # otherwise, tests fail + r.status_code = 0 # with this status_code, content returns None return r return function(method, url, **kwargs) return wrapped diff --git a/requests/sessions.py b/requests/sessions.py index 15055915..cd6daa66 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -244,7 +244,6 @@ class Session(object): # Return the response. return r.response - def get(self, url, **kwargs): """Sends a GET request. Returns :class:`Response` object. @@ -255,7 +254,6 @@ class Session(object): kwargs.setdefault('allow_redirects', True) return self.request('get', url, **kwargs) - def options(self, url, **kwargs): """Sends a OPTIONS request. Returns :class:`Response` object. @@ -266,7 +264,6 @@ class Session(object): kwargs.setdefault('allow_redirects', True) return self.request('options', url, **kwargs) - def head(self, url, **kwargs): """Sends a HEAD request. Returns :class:`Response` object. @@ -277,7 +274,6 @@ class Session(object): kwargs.setdefault('allow_redirects', False) return self.request('head', url, **kwargs) - def post(self, url, data=None, **kwargs): """Sends a POST request. Returns :class:`Response` object. @@ -288,7 +284,6 @@ class Session(object): return self.request('post', url, data=data, **kwargs) - def put(self, url, data=None, **kwargs): """Sends a PUT request. Returns :class:`Response` object. @@ -299,7 +294,6 @@ class Session(object): return self.request('put', url, data=data, **kwargs) - def patch(self, url, data=None, **kwargs): """Sends a PATCH request. Returns :class:`Response` object. @@ -310,7 +304,6 @@ class Session(object): return self.request('patch', url, data=data, **kwargs) - def delete(self, url, **kwargs): """Sends a DELETE request. Returns :class:`Response` object. diff --git a/requests/status_codes.py b/requests/status_codes.py index da74286d..e25ecdb9 100644 --- a/requests/status_codes.py +++ b/requests/status_codes.py @@ -83,4 +83,4 @@ for (code, titles) in list(_codes.items()): for title in titles: setattr(codes, title, code) if not title.startswith('\\'): - setattr(codes, title.upper(), code) \ No newline at end of file + setattr(codes, title.upper(), code) diff --git a/requests/structures.py b/requests/structures.py index fd1051a8..3fda9843 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -47,6 +47,7 @@ class CaseInsensitiveDict(dict): else: return default + class LookupDict(dict): """Dictionary lookup object.""" diff --git a/requests/utils.py b/requests/utils.py index 53bb80f5..9b8ea21d 100644 --- a/requests/utils.py +++ b/requests/utils.py @@ -49,6 +49,7 @@ POSSIBLE_CA_BUNDLE_PATHS = [ '/etc/ssl/ca-bundle.pem', ] + def get_os_ca_bundle_path(): """Try to pick an available CA certificate bundle provided by the OS.""" for path in POSSIBLE_CA_BUNDLE_PATHS: @@ -60,6 +61,7 @@ def get_os_ca_bundle_path(): # otherwise, try and use the OS bundle DEFAULT_CA_BUNDLE_PATH = CERTIFI_BUNDLE_PATH or get_os_ca_bundle_path() + def dict_to_sequence(d): """Returns an internal sequence dictionary update.""" @@ -472,6 +474,7 @@ def requote_uri(uri): # or '%') return quote(unquote_unreserved(uri), safe="!#$%&'()*+,/:;=?@[]~") + def get_environ_proxies(): """Return a dict of environment proxies.""" diff --git a/tests/informal/test_leaked_connections.py b/tests/informal/test_leaked_connections.py index 5357bf2f..438a6cee 100644 --- a/tests/informal/test_leaked_connections.py +++ b/tests/informal/test_leaked_connections.py @@ -6,6 +6,7 @@ the body of the request is not read. import gc, os, subprocess, requests, sys + def main(): gc.disable() diff --git a/tests/test_cookies.py b/tests/test_cookies.py index c6f71b42..e1c4203a 100755 --- a/tests/test_cookies.py +++ b/tests/test_cookies.py @@ -16,6 +16,7 @@ from requests.compat import cookielib sys.path.append('.') from test_requests import httpbin, TestBaseMixin + class CookieTests(TestBaseMixin, unittest.TestCase): def test_cookies_from_response(self): @@ -106,22 +107,22 @@ class CookieTests(TestBaseMixin, unittest.TestCase): def test_disabled_cookie_persistence(self): """Test that cookies are not persisted when configured accordingly.""" - config = {'store_cookies' : False} + config = {'store_cookies': False} # Check the case when no cookie is passed as part of the request and the one in response is ignored - cookies = requests.get(httpbin('cookies', 'set', 'key', 'value'), config = config).cookies + cookies = requests.get(httpbin('cookies', 'set', 'key', 'value'), config=config).cookies self.assertTrue(cookies.get("key") is None) # Test that the cookies passed while making the request still gets used and is available in response object. # only the ones received from server is not saved - cookies_2 = requests.get(httpbin('cookies', 'set', 'key', 'value'), config = config,\ - cookies = {"key_2" : "value_2"}).cookies + cookies_2 = requests.get(httpbin('cookies', 'set', 'key', 'value'), config=config,\ + cookies={"key_2": "value_2"}).cookies self.assertEqual(len(cookies_2), 1) self.assertEqual(cookies_2.get("key_2"), "value_2") # Use the session and make sure that the received cookie is not used in subsequent calls s = requests.session() - s.get(httpbin('cookies', 'set', 'key', 'value'), config = config) + s.get(httpbin('cookies', 'set', 'key', 'value'), config=config) r = s.get(httpbin('cookies')) self.assertEqual(json.loads(r.text)['cookies'], {}) @@ -134,7 +135,7 @@ class CookieTests(TestBaseMixin, unittest.TestCase): self.assertEqual(len(c), len(r.cookies.keys())) self.assertEqual(len(c), len(r.cookies.values())) self.assertEqual(len(c), len(r.cookies.items())) - + # domain and path utility functions domain = r.cookies.list_domains()[0] path = r.cookies.list_paths()[0] @@ -151,13 +152,14 @@ class CookieTests(TestBaseMixin, unittest.TestCase): # test keys, values, and items self.assertEqual(r.cookies.keys(), ['myname']) self.assertEqual(r.cookies.values(), ['myvalue']) - self.assertEqual(r.cookies.items(), [('myname','myvalue')]) - + self.assertEqual(r.cookies.items(), [('myname', 'myvalue')]) + # test if we can convert jar to dict dictOfCookies = dict(r.cookies) - self.assertEqual(dictOfCookies, {'myname':'myvalue'}) + self.assertEqual(dictOfCookies, {'myname': 'myvalue'}) self.assertEqual(dictOfCookies, r.cookies.get_dict()) + class LWPCookieJarTest(TestBaseMixin, unittest.TestCase): """Check store/load of cookies to FileCookieJar's, specifically LWPCookieJar's.""" @@ -254,6 +256,7 @@ class LWPCookieJarTest(TestBaseMixin, unittest.TestCase): self.assertEqual(len(cookiejar_2), 1) self.assertCookieHas(list(cookiejar_2)[0], name='Persistent', value='CookiesAreScary') + class MozCookieJarTest(LWPCookieJarTest): """Same test, but substitute MozillaCookieJar.""" diff --git a/tests/test_requests.py b/tests/test_requests.py index 0b218318..10b43deb 100755 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -7,7 +7,6 @@ import sys import os sys.path.insert(0, os.path.abspath('..')) - import json import os import unittest @@ -20,6 +19,7 @@ from requests.compat import str, StringIO from requests import HTTPError from requests import get, post, head, put from requests.auth import HTTPBasicAuth, HTTPDigestAuth +from requests.exceptions import InvalidURL if 'HTTPBIN_URL' not in os.environ: os.environ['HTTPBIN_URL'] = 'http://httpbin.org/' @@ -856,6 +856,19 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): assert ds1.prefetch assert not ds2.prefetch + def test_connection_error(self): + try: + get('http://localhost:1/nope') + except requests.ConnectionError: + pass + else: + assert False + + def test_connection_error_with_safe_mode(self): + config = {'safe_mode': True} + r = get('http://localhost:1/nope', allow_redirects=False, config=config) + assert r.content == None + # def test_invalid_content(self): # # WARNING: if you're using a terrible DNS provider (comcast), # # this will fail. @@ -1019,10 +1032,10 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): list for a value in the data argument.""" data = {'field': ['a', 'b']} - files = {'file': 'Garbled data'} + files = {'field': 'Garbled data'} r = post(httpbin('post'), data=data, files=files) t = json.loads(r.text) - self.assertEqual(t.get('form'), {'field': 'a, b'}) + self.assertEqual(t.get('form'), {'field': ['a', 'b']}) self.assertEqual(t.get('files'), files) r = post(httpbin('post'), data=data, files=files.items()) t = r.json @@ -1035,6 +1048,80 @@ class RequestsTestSuite(TestSetup, TestBaseMixin, unittest.TestCase): t = json.loads(r.text) self.assertEqual(t.get('headers').get('Content-Type'), '') + def test_prefetch_redirect_bug(self): + """Test that prefetch persists across redirections.""" + res = get(httpbin('redirect/2'), prefetch=False) + # prefetch should persist across the redirect; if it doesn't, + # this attempt to iterate will crash because the content has already + # been read. + first_line = next(res.iter_lines()) + self.assertTrue(first_line.strip().decode('utf-8').startswith('{')) + + def test_prefetch_return_response_interaction(self): + """Test that prefetch can be overridden as a kwarg to `send`.""" + req = requests.get(httpbin('get'), return_response=False) + req.send(prefetch=False) + # content should not have been prefetched, and iter_lines should succeed + first_line = next(req.response.iter_lines()) + self.assertTrue(first_line.strip().decode('utf-8').startswith('{')) + + def test_accept_objects_with_string_representations_as_urls(self): + """Test that URLs can be set to objects with string representations, + e.g. for use with furl.""" + class URL(): + def __unicode__(self): + # Can't have unicode literals in Python3, so avoid them. + # TODO: fixup when moving to Python 3.3 + if (sys.version_info[0] == 2): + return 'http://httpbin.org/get'.decode('utf-8') + else: + return 'http://httpbin.org/get' + + def __str__(self): + return 'http://httpbin.org/get' + + r = get(URL()) + self.assertEqual(r.status_code, 200) + + def test_post_fields_with_multiple_values_and_files_as_tuples(self): + """Test that it is possible to POST multiple data and file fields + with the same name. + https://github.com/kennethreitz/requests/pull/746 + """ + + fields = [ + ('__field__', '__value__'), + ('__field__', '__value__'), + ] + + r = post(httpbin('post'), data=fields, files=fields) + t = json.loads(r.text) + + self.assertEqual(t.get('form'), { + '__field__': [ + '__value__', + '__value__', + ] + }) + + # It's not currently possible to test for multiple file fields with + # the same name against httpbin so we need to inspect the encoded + # body manually. + request = r.request + body, content_type = request._encode_files(request.files) + file_field = (b'Content-Disposition: form-data;' + b' name="__field__"; filename="__field__"') + self.assertEqual(body.count(b'__value__'), 4) + self.assertEqual(body.count(file_field), 2) + + def test_bytes_files(self): + """Test that `bytes` can be used as the values of `files`.""" + post(httpbin('post'), files={'test': b'test'}) + + def test_invalid_urls_throw_requests_exception(self): + """Test that URLs with invalid labels throw + Requests.exceptions.InvalidURL instead of UnicodeError.""" + self.assertRaises(InvalidURL, get, 'http://.google.com/') if __name__ == '__main__': unittest.main() diff --git a/tests/test_requests_ext.py b/tests/test_requests_ext.py index 883bdceb..3e0d5b73 100644 --- a/tests/test_requests_ext.py +++ b/tests/test_requests_ext.py @@ -25,17 +25,14 @@ class RequestsTestSuite(unittest.TestCase): def test_addition(self): assert (1 + 1) == 2 - def test_ssl_hostname_ok(self): requests.get('https://github.com', verify=True) - def test_ssl_hostname_not_ok(self): requests.get('https://kennethreitz.com', verify=False) self.assertRaises(requests.exceptions.SSLError, requests.get, 'https://kennethreitz.com') - def test_ssl_hostname_session_not_ok(self): s = requests.session() @@ -44,7 +41,6 @@ class RequestsTestSuite(unittest.TestCase): s.get('https://kennethreitz.com', verify=False) - def test_binary_post(self): '''We need to be careful how we build the utf-8 string since unicode literals are a syntax error in python3 @@ -59,13 +55,10 @@ class RequestsTestSuite(unittest.TestCase): raise EnvironmentError('Flesh out this test for your environment.') requests.post('http://www.google.com/', data=utf8_string) - - def test_unicode_error(self): url = 'http://blip.fm/~1abvfu' requests.get(url) - def test_chunked_head_redirect(self): url = "http://t.co/NFrx0zLG" r = requests.head(url, allow_redirects=True) @@ -110,7 +103,7 @@ class RequestsTestSuite(unittest.TestCase): s = requests.session() s.get(url='http://tinyurl.com/preview.php?disable=1') # we should have set a cookie for tinyurl: preview=0 - self.assertIn('preview', s.cookies) + self.assertTrue('preview' in s.cookies) self.assertEqual(s.cookies['preview'], '0') self.assertEqual(list(s.cookies)[0].name, 'preview') self.assertEqual(list(s.cookies)[0].domain, 'tinyurl.com') @@ -118,14 +111,13 @@ class RequestsTestSuite(unittest.TestCase): # get cookies on another domain r2 = s.get(url='http://httpbin.org/cookies') # the cookie is not there - self.assertNotIn('preview', json.loads(r2.text)['cookies']) + self.assertTrue('preview' not in json.loads(r2.text)['cookies']) # this redirects to another domain, httpbin.org # cookies of the first domain should NOT be sent to the next one r3 = s.get(url='http://tinyurl.com/7zp3jnr') assert r3.url == 'http://httpbin.org/cookies' - self.assertNotIn('preview', json.loads(r2.text)['cookies']) + self.assertTrue('preview' not in json.loads(r2.text)['cookies']) if __name__ == '__main__': unittest.main() - diff --git a/tests/test_requests_https.py b/tests/test_requests_https.py index c6ea8f35..1691a8c0 100755 --- a/tests/test_requests_https.py +++ b/tests/test_requests_https.py @@ -9,6 +9,7 @@ import unittest sys.path.insert(0, os.path.abspath('..')) import requests + class HTTPSTest(unittest.TestCase): """Smoke test for https functionality."""