diff --git a/AUTHORS b/AUTHORS index 280977da..224dda26 100644 --- a/AUTHORS +++ b/AUTHORS @@ -18,4 +18,11 @@ Patches and Suggestions - Rob Madole - Aram Dulyan - Johannes Gorset -- 村山めがね (Megane Murayama) \ No newline at end of file +- 村山めがね (Megane Murayama) +- James Rowe +- Daniel Schauenberg +- Zbigniew Siciarz +- Daniele Tricoli 'Eriol' +- Richard Boulton +- Miguel Olivares +- Alberto Paro \ No newline at end of file diff --git a/HISTORY.rst b/HISTORY.rst index c119319b..dd7f97e2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,19 @@ History ------- +0.5.0 (2011-06-21) +++++++++++++++++++ + +* PATCH Support +* Support for Proxies +* HTTPBin Test Suite +* Redirect Fixes +* settings.verbose stream writing +* Querystrings for all methods +* URLErrors (Connection Refused, Timeout, Invalid URLs) are treated as explicity raised + ``r.requests.get('hwe://blah'); r.raise_for_status()`` + + 0.4.1 (2011-05-22) ++++++++++++++++++ diff --git a/README.rst b/README.rst index ce4eb581..d73a48f3 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -Requests: The Simple (e.g. usable) HTTP Module -============================================== +Requests: HTTP for Humans +========================= Most existing Python modules for dealing HTTP requests are insane. I have to look up *everything* that I want to do. Most of my worst Python experiences are a result of the various built-in HTTP libraries (yes, even worse than Logging). @@ -10,12 +10,13 @@ Really simple. Features -------- -- Extremely simple GET, HEAD, POST, PUT, DELETE Requests +- Extremely simple HEAD, GET, POST, PUT, PATCH, DELETE Requests + Simple HTTP Header Request Attachment + Simple Data/Params Request Attachment + Simple Multipart File Uploads + CookieJar Support + Redirection History + + Proxy Support + Redirection Recursion Urllib Fix + Auto Decompression of GZipped Content + Unicode URL Support @@ -35,15 +36,14 @@ It couldn't be simpler. :: HTTPS? Basic Authentication? :: - >>> r = requests.get('https://convore.com/api/account/verify.json') + >>> r = requests.get('https://httpbin.ep.ip/basic-auth/user/pass') >>> r.status_code 401 Uh oh, we're not authorized! Let's add authentication. :: - >>> conv_auth = ('requeststest', 'requeststest') - >>> r = requests.get('https://convore.com/api/account/verify.json', auth=conv_auth) + >>> r = requests.get(https://httpbin.ep.ip/basic-auth/user/pass', auth=('user', 'pass')) >>> r.status_code 200 @@ -52,7 +52,7 @@ Uh oh, we're not authorized! Let's add authentication. :: 'application/json' >>> r.content - '{"username": "requeststest", "url": "/users/requeststest/", "id": "9408", "img": "censored-long-url"}' + '{"authenticated": true, "user": "user"}' @@ -66,24 +66,28 @@ All request functions return a Response object (see below). If a {filename: fileobject} dictionary is passed in (files=...), a multipart_encode upload will be performed. If CookieJar object is is passed in (cookies=...), the cookies will be sent with the request. - GET Requests - >>> requests.get(url, params={}, headers={}, cookies=None, auth=None) - - HEAD Requests - >>> requests.head(url, params={}, headers={}, cookies=None, auth=None) + >>> requests.head(url, params={}, headers={}, cookies=None, auth=None, timeout=None, proxies={}) - PUT Requests - >>> requests.put(url, data='', headers={}, files={}, cookies=None, auth=None) + GET Requests + >>> requests.get(url, params={}, headers={}, cookies=None, auth=None, timeout=None, proxies={}) POST Requests - >>> requests.post(url, data={}, headers={}, files={}, cookies=None, auth=None) + >>> requests.post(url, data={}, headers={}, files={}, cookies=None, auth=None, timeout=None, allow_redirects=False, params{}, proxies={}) + + + PUT Requests + >>> requests.put(url, data={}, headers={}, files={}, cookies=None, auth=None, timeout=None, allow_redirects=False, params{}, proxies={}) + + + PATCH Requests + >>> requests.post(url, data={}, headers={}, files={}, cookies=None, auth=None, timeout=None, allow_redirects=False, params{}, proxies={}) DELETE Requests - >>> requests.delete(url, params={}, headers={}, cookies=None, auth=None) + >>> requests.delete(url, params={}, headers={}, cookies=None, auth=None, timeout=None, allow_redirects=False, params{}, proxies={}) diff --git a/docs/_themes/kr/layout.html b/docs/_themes/kr/layout.html index abb6f8a8..623e3ef0 100644 --- a/docs/_themes/kr/layout.html +++ b/docs/_themes/kr/layout.html @@ -30,4 +30,18 @@ })(); + + {%- endblock %} diff --git a/docs/api.rst b/docs/api.rst index 9d9bc0a8..06939f19 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -16,11 +16,13 @@ Main Interface All of Request's functionality can be accessed by these 5 methods. They all return a :class:`Response ` object. +.. autofunction:: head .. autofunction:: get .. autofunction:: post .. autofunction:: put +.. autofunction:: patch .. autofunction:: delete -.. autofunction:: head + ----------- diff --git a/docs/index.rst b/docs/index.rst index c270663b..663bda58 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -27,8 +27,8 @@ Things shouldn’t be this way. Not in Python. See `the same code, without Requests `_. -Requests allow you to send **GET**, **HEAD**, **PUT**, -**POST**, and **DELETE** HTTP requests. You can add headers, form data, +Requests allow you to send **HEAD**, **GET**, **POST**, **PUT**, +**PATCH**, and **DELETE** HTTP requests. You can add headers, form data, multipart files, and parameters with simple Python dictionaries, and access the response data in the same way. It's powered by :py:class:`urllib2`, but it does all the hard work and crazy hacks for you. @@ -36,7 +36,7 @@ all the hard work and crazy hacks for you. Testimonals ----------- -`Twitter, Inc `_ and `The Library of Congress `_ use Requests internally. +`Twitter, Inc `_ uses Requests internally. **Daniel Greenfeld** Nuked a 1200 LOC spaghetti code library with 10 lines of code thanks to @kennethreitz's request library. Today has been AWESOME. diff --git a/requests/api.py b/requests/api.py index 0e274101..8e328d2b 100644 --- a/requests/api.py +++ b/requests/api.py @@ -11,37 +11,43 @@ This module impliments the Requests API. """ -import requests import config from .models import Request, Response, AuthManager, AuthObject, auth_manager -__all__ = ('request', 'get', 'head', 'post', 'put', 'delete') +__all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete') +def request(method, url, + params=None, data=None, headers=None, cookies=None, files=None, auth=None, + timeout=None, allow_redirects=False, proxies=None): - -def request(method, url, **kwargs): """Constructs and sends a :class:`Request `. Returns :class:`Response ` object. :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary of GET/HEAD/DELETE Parameters to send with the :class:`Request`. - :param data: (optional) Bytes/Dictionary of PUT/POST Data to send with the :class:`Request`. + :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - data = kwargs.pop('data', dict()) or kwargs.pop('params', dict()) - r = Request(method=method, url=url, data=data, headers=kwargs.pop('headers', dict()), - cookiejar=kwargs.get('cookies', None), - files=kwargs.get('files', None), - auth=kwargs.get('auth', auth_manager.get_auth(url)), - timeout=kwargs.get('timeout', config.settings.timeout), - allow_redirects=kwargs.get('allow_redirects', None) + r = Request( + method = method, + url = url, + data = data, + params = params, + headers = headers, + cookiejar = cookies, + files = files, + auth = auth or auth_manager.get_auth(url), + timeout = timeout or config.settings.timeout, + allow_redirects = allow_redirects, + proxies = proxies ) r.send() @@ -49,73 +55,130 @@ def request(method, url, **kwargs): return r.response -def get(url, params={}, headers={}, cookies=None, auth=None, **kwargs): +def get(url, + params=None, headers=None, cookies=None, auth=None, timeout=None, + proxies=None): + """Sends a GET request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary of GET Parameters to send with the :class:`Request`. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - return request('GET', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs) + return request('GET', url, + params=params, headers=headers, cookies=cookies, auth=auth, + timeout=timeout, proxies=proxies) -def head(url, params={}, headers={}, cookies=None, auth=None, **kwargs): +def head(url, + params=None, headers=None, cookies=None, auth=None, timeout=None, + proxies=None): + """Sends a HEAD request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary of GET Parameters to send with the :class:`Request`. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - return request('HEAD', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs) + return request('HEAD', url, + params=params, headers=headers, cookies=cookies, auth=auth, + timeout=timeout, proxies=proxies) -def post(url, data={}, headers={}, files=None, cookies=None, auth=None, **kwargs): +def post(url, + data='', headers=None, files=None, cookies=None, auth=None, timeout=None, + allow_redirects=False, params=None, proxies=None): + """Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary of POST data to send with the :class:`Request`. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - return request('POST', url, data=data, headers=headers, files=files, cookies=cookies, auth=auth, **kwargs) + return request('POST', url, + params=params, data=data, headers=headers, files=files, + cookies=cookies, auth=auth, timeout=timeout, + allow_redirects=allow_redirects, proxies=proxies) -def put(url, data='', headers={}, files={}, cookies=None, auth=None, **kwargs): +def put(url, data='', headers=None, files=None, cookies=None, auth=None, + timeout=None, allow_redirects=False, params=None, proxies=None): """Sends a PUT request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Bytes of PUT Data to send with the :class:`Request`. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - return request('PUT', url, data=data, headers=headers, files=files, cookies=cookies, auth=auth, **kwargs) + return request('PUT', url, + params=params, data=data, headers=headers, files=files, + cookies=cookies, auth=auth, timeout=timeout, + allow_redirects=allow_redirects, proxies=proxies) -def delete(url, params={}, headers={}, cookies=None, auth=None, **kwargs): +def patch(url, data='', headers=None, files=None, cookies=None, auth=None, + timeout=None, allow_redirects=False, params=None, proxies=None): + """Sends a PATCH request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. + :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. + :param cookies: (optional) CookieJar object to send with the :class:`Request`. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + """ + + return request('PATCH', url, + params=params, data=data, headers=headers, files=files, + cookies=cookies, auth=auth, timeout=timeout, + allow_redirects=allow_redirects, proxies=proxies) + + +def delete(url, + params=None, headers=None, cookies=None, auth=None, timeout=None, + allow_redirects=False, proxies=None): + """Sends a DELETE request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary of DELETE Parameters to send with the :class:`Request`. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. :param cookies: (optional) CookieJar object to send with the :class:`Request`. :param auth: (optional) AuthObject to enable Basic HTTP Auth. :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. """ - return request('DELETE', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs) + return request('DELETE', url, + params=params, headers=headers, cookies=cookies, auth=auth, + timeout=timeout, allow_redirects=allow_redirects, proxies=proxies) diff --git a/requests/config.py b/requests/config.py index 63d3fa99..0878da92 100644 --- a/requests/config.py +++ b/requests/config.py @@ -12,7 +12,7 @@ class Settings(object): _singleton = {} # attributes with defaults - __attrs__ = ('timeout',) + __attrs__ = ('timeout', 'verbose') def __init__(self, **kwargs): super(Settings, self).__init__() diff --git a/requests/core.py b/requests/core.py index 7f3d723a..87f55e45 100644 --- a/requests/core.py +++ b/requests/core.py @@ -12,8 +12,8 @@ This module implements the main Requests system. """ __title__ = 'requests' -__version__ = '0.4.1' -__build__ = 0x000401 +__version__ = '0.5.0' +__build__ = 0x000500 __author__ = 'Kenneth Reitz' __license__ = 'ISC' __copyright__ = 'Copyright 2011 Kenneth Reitz' diff --git a/requests/models.py b/requests/models.py index 2c3241d4..099f1c66 100644 --- a/requests/models.py +++ b/requests/models.py @@ -6,7 +6,6 @@ requests.models """ -import requests import urllib import urllib2 import socket @@ -14,7 +13,9 @@ import zlib from urllib2 import HTTPError from urlparse import urlparse +from datetime import datetime +from .config import settings from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler from .structures import CaseInsensitiveDict from .packages.poster.encode import multipart_encode @@ -22,17 +23,20 @@ from .packages.poster.streaminghttp import register_openers, get_handlers from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod +REDIRECT_STATI = (301, 302, 303, 307) + class Request(object): """The :class:`Request ` object. It carries out all functionality of Requests. Recommended interface is with the Requests functions. """ - _METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE') + _METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE', 'PATCH') - def __init__(self, url=None, headers=dict(), files=None, method=None, - data=dict(), auth=None, cookiejar=None, timeout=None, - redirect=True, allow_redirects=False): + def __init__(self, + url=None, headers=dict(), files=None, method=None, data=dict(), + params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False, + allow_redirects=False, proxies=None): socket.setdefaulttimeout(timeout) @@ -44,23 +48,22 @@ class Request(object): self.files = files #: HTTP Method to use. Available: GET, HEAD, PUT, POST, DELETE. self.method = method - #: Form or Byte data to attach to the :class:`Request `. - self.data = dict() + #: Dictionary or byte of request body data to attach to the + #: :class:`Request `. + self.data = None + #: Dictionary or byte of querystring data to attach to the + #: :class:`Request `. + self.params = None #: True if :class:`Request ` is part of a redirect chain (disables history #: and HTTPError storage). self.redirect = redirect #: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``) self.allow_redirects = allow_redirects + # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) + self.proxies = proxies - if hasattr(data, 'items'): - for (k, v) in data.items(): - self.data.update({ - k.encode('utf-8') if isinstance(k, unicode) else k: - v.encode('utf-8') if isinstance(v, unicode) else v - }) - self._enc_data = urllib.urlencode(self.data) - else: - self._enc_data = self.data = data + self.data, self._enc_data = self._encode_params(data) + self.params, self._enc_params = self._encode_params(params) #: :class:`Response ` instance, containing #: content and metadata of HTTP Response, once :attr:`sent `. @@ -113,6 +116,8 @@ class Request(object): _handlers.append(self.auth.handler) + if self.proxies: + _handlers.append(urllib2.ProxyHandler(self.proxies)) _handlers.append(HTTPRedirectHandler) @@ -135,6 +140,7 @@ class Request(object): return opener.open + def _build_response(self, resp): """Build internal :class:`Response ` object from given response.""" @@ -155,6 +161,8 @@ class Request(object): except zlib.error: pass + # TODO: Support deflate + response.url = getattr(resp, 'url', None) return response @@ -164,6 +172,9 @@ class Request(object): r = build(resp) + if r.status_code in REDIRECT_STATI: + self.redirect = True + if self.redirect: while ( @@ -177,7 +188,7 @@ class Request(object): url = r.headers['location'] - # Facilitate for non-RFC2616-compliant 'location' headers + # Facilitate non-RFC2616-compliant 'location' headers # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') if not urlparse(url).netloc: parent_url_components = urlparse(self.url) @@ -191,7 +202,8 @@ class Request(object): request = Request( url, self.headers, self.files, method, - self.data, self.auth, self.cookiejar, redirect=False + self.data, self.params, self.auth, self.cookiejar, + redirect=True ) request.send() r = request.response @@ -202,16 +214,37 @@ class Request(object): @staticmethod - def _build_url(url, data=None): - """Build URLs.""" + def _encode_params(data): + """Encode parameters in a piece of data. - if urlparse(url).query: - return '%s&%s' % (url, data) + If the data supplied is a dictionary, encodes each parameter in it, and + returns the dictionary of encoded parameters, and a urlencoded version + of that. + + Otherwise, assumes the data is already encoded appropriately, and + returns it twice. + + """ + if hasattr(data, 'items'): + result = {} + for (k, v) in data.items(): + result[k.encode('utf-8') if isinstance(k, unicode) else k] \ + = v.encode('utf-8') if isinstance(v, unicode) else v + return result, urllib.urlencode(result) else: - if data: - return '%s?%s' % (url, data) + return data, data + + + def _build_url(self): + """Build the actual URL to use""" + + if self._enc_params: + if urlparse(self.url).query: + return '%s&%s' % (self.url, self._enc_params) else: - return url + return '%s?%s' % (self.url, self._enc_params) + else: + return self.url def send(self, anyway=False): @@ -227,8 +260,16 @@ class Request(object): self._checks() success = False + # Logging + if settings.verbose: + settings.verbose.write('%s %s %s\n' % ( + datetime.now().isoformat(), self.method, self.url + )) + + + url = self._build_url() if self.method in ('GET', 'HEAD', 'DELETE'): - req = _Request(self._build_url(self.url, self._enc_data), method=self.method) + req = _Request(url, method=self.method) else: if self.files: @@ -238,10 +279,10 @@ class Request(object): self.files.update(self.data) datagen, headers = multipart_encode(self.files) - req = _Request(self.url, data=datagen, headers=headers, method=self.method) + req = _Request(url, data=datagen, headers=headers, method=self.method) else: - req = _Request(self.url, data=self._enc_data, method=self.method) + req = _Request(url, data=self._enc_data, method=self.method) if self.headers: req.headers.update(self.headers) @@ -255,12 +296,15 @@ class Request(object): if self.cookiejar is not None: self.cookiejar.extract_cookies(resp, req) - except urllib2.HTTPError, why: + except (urllib2.HTTPError, urllib2.URLError), why: + if hasattr(why, 'reason'): + if isinstance(why.reason, socket.timeout): + why = Timeout(why) + self._build_response(why) if not self.redirect: self.response.error = why - except urllib2.URLError, error: - raise Timeout if isinstance(error.reason, socket.timeout) else error + else: self._build_response(resp) self.response.ok = True @@ -271,6 +315,7 @@ class Request(object): self.sent = self.response.ok + return self.sent diff --git a/requests/monkeys.py b/requests/monkeys.py index b8fe5041..41cd3706 100644 --- a/requests/monkeys.py +++ b/requests/monkeys.py @@ -26,7 +26,6 @@ class Request(urllib2.Request): return urllib2.Request.get_method(self) - class HTTPRedirectHandler(urllib2.HTTPRedirectHandler): def http_error_301(self, req, fp, code, msg, headers): diff --git a/requests/structures.py b/requests/structures.py index 0c82c7b4..bfee7b19 100644 --- a/requests/structures.py +++ b/requests/structures.py @@ -8,30 +8,14 @@ Datastructures that power Requests. """ -from UserDict import DictMixin - - -class CaseInsensitiveDict(DictMixin): +class CaseInsensitiveDict(dict): """Case-insensitive Dictionary for :class:`Response ` Headers. For example, ``headers['content-encoding']`` will return the value of a ``'Content-Encoding'`` response header.""" - def __init__(self, *args, **kwargs): - # super(CaseInsensitiveDict, self).__init__() - self.data = dict(*args, **kwargs) - - def __repr__(self): - return self.data.__repr__() - - def __getstate__(self): - return self.data.copy() - - def __setstate__(self, d): - self.data = d - def _lower_keys(self): - return map(str.lower, self.data.keys()) + return map(str.lower, self.keys()) def __contains__(self, key): @@ -39,26 +23,6 @@ class CaseInsensitiveDict(DictMixin): def __getitem__(self, key): - - if key.lower() in self: + # We allow fall-through here, so values default to None + if key in self: return self.items()[self._lower_keys().index(key.lower())][1] - - - def __setitem__(self, key, value): - return self.data.__setitem__(key, value) - - - def __delitem__(self, key): - return self.data.__delitem__(key) - - - def __keys__(self): - return self.data.__keys__() - - - def __iter__(self): - return self.data.__iter__() - - - def iteritems(self): - return self.data.iteritems() diff --git a/setup.py b/setup.py index f0cebcd5..1b15bc15 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,10 @@ import os import sys import requests -from distutils.core import setup +try: + from setuptools import setup +except ImportError: + from distutils.core import setup diff --git a/test_requests.py b/test_requests.py index 358dfce6..848297e4 100755 --- a/test_requests.py +++ b/test_requests.py @@ -6,50 +6,84 @@ from __future__ import with_statement import unittest import cookielib +import omnijson as json + import requests + +HTTPBIN_URL = 'http://httpbin.org/' +HTTPSBIN_URL = 'https://httpbin.ep.io/' + +# HTTPBIN_URL = 'http://staging.httpbin.org/' +# HTTPSBIN_URL = 'https://httpbin-staging.ep.io/' + + +def httpbin(*suffix): + """Returns url for HTTPBIN resource.""" + + return HTTPBIN_URL + '/'.join(suffix) + + +def httpsbin(*suffix): + """Returns url for HTTPSBIN resource.""" + + return HTTPSBIN_URL + '/'.join(suffix) + + + class RequestsTestSuite(unittest.TestCase): """Requests test cases.""" + def setUp(self): pass + def tearDown(self): """Teardown.""" pass + def test_invalid_url(self): self.assertRaises(ValueError, requests.get, 'hiwpefhipowhefopw') + def test_HTTP_200_OK_GET(self): - r = requests.get('http://google.com') + r = requests.get(httpbin('/')) self.assertEqual(r.status_code, 200) + def test_HTTPS_200_OK_GET(self): - r = requests.get('https://google.com') + r = requests.get(httpsbin('/')) self.assertEqual(r.status_code, 200) + def test_HTTP_200_OK_GET_WITH_PARAMS(self): heads = {'User-agent': 'Mozilla/5.0'} - r = requests.get('http://www.google.com/search', params={'q': 'test'}, headers=heads) + r = requests.get(httpbin('user-agent'), headers=heads) + + assert heads['User-agent'] in r.content self.assertEqual(r.status_code, 200) + def test_HTTP_200_OK_GET_WITH_MIXED_PARAMS(self): heads = {'User-agent': 'Mozilla/5.0'} - r = requests.get('http://google.com/search?test=true', params={'q': 'test'}, headers=heads) + r = requests.get(httpbin('get') + '?test=true', params={'q': 'test'}, headers=heads) self.assertEqual(r.status_code, 200) + def test_user_agent_transfers(self): """Issue XX""" + heads = { 'User-agent': 'Mozilla/5.0 (github.com/kennethreitz/requests)' } - r = requests.get('http://whatsmyua.com', headers=heads); + r = requests.get(httpbin('user-agent'), headers=heads); self.assertTrue(heads['User-agent'] in r.content) heads = { @@ -57,20 +91,43 @@ class RequestsTestSuite(unittest.TestCase): 'Mozilla/5.0 (github.com/kennethreitz/requests)' } - r = requests.get('http://whatsmyua.com', headers=heads); + r = requests.get(httpbin('user-agent'), headers=heads); self.assertTrue(heads['user-agent'] in r.content) + def test_HTTP_200_OK_HEAD(self): - r = requests.head('http://google.com') + r = requests.head(httpbin('/')) self.assertEqual(r.status_code, 200) + def test_HTTPS_200_OK_HEAD(self): - r = requests.head('https://google.com') + r = requests.head(httpsbin('/')) self.assertEqual(r.status_code, 200) + + def test_HTTP_200_OK_PUT(self): + r = requests.put(httpbin('put')) + self.assertEqual(r.status_code, 200) + + + def test_HTTPS_200_OK_PUT(self): + r = requests.put(httpsbin('put')) + self.assertEqual(r.status_code, 200) + + + def test_HTTP_200_OK_PATCH(self): + r = requests.patch(httpbin('patch')) + self.assertEqual(r.status_code, 200) + + + def test_HTTPS_200_OK_PATCH(self): + r = requests.patch(httpsbin('patch')) + self.assertEqual(r.status_code, 200) + + def test_AUTH_HTTPS_200_OK_GET(self): - auth = ('requeststest', 'requeststest') - url = 'https://convore.com/api/account/verify.json' + auth = ('user', 'pass') + url = httpsbin('basic-auth', 'user', 'pass') r = requests.get(url, auth=auth) self.assertEqual(r.status_code, 200) @@ -81,108 +138,188 @@ class RequestsTestSuite(unittest.TestCase): # reset auto authentication requests.auth_manager.empty() + def test_POSTBIN_GET_POST_FILES(self): - bin = requests.post('http://www.postbin.org/') - self.assertEqual(bin.status_code, 302) + url = httpbin('post') + post = requests.post(url).raise_for_status() - post_url = bin.headers['location'] - post = requests.post(post_url, data={'some': 'data'}) - self.assertEqual(post.status_code, 201) + post = requests.post(url, data={'some': 'data'}) + self.assertEqual(post.status_code, 200) - post2 = requests.post(post_url, files={'some': open('test_requests.py')}) - self.assertEqual(post2.status_code, 201) + post2 = requests.post(url, files={'some': open('test_requests.py')}) + self.assertEqual(post2.status_code, 200) - post3 = requests.post(post_url, data='[{"some": "json"}]') - self.assertEqual(post.status_code, 201) + post3 = requests.post(url, data='[{"some": "json"}]') + self.assertEqual(post3.status_code, 200) def test_POSTBIN_GET_POST_FILES_WITH_PARAMS(self): - bin = requests.post('http://www.postbin.org/') - self.assertEqual(bin.status_code, 302) - post_url = bin.headers['location'] - - post2 = requests.post(post_url, files={'some': open('test_requests.py')}, data={'some': 'data'}) - self.assertEqual(post2.status_code, 201) + url = httpbin('post') + post = requests.post(url, files={'some': open('test_requests.py')}, data={'some': 'data'}) + self.assertEqual(post.status_code, 200) def test_POSTBIN_GET_POST_FILES_WITH_HEADERS(self): - bin = requests.post('http://www.postbin.org/') - self.assertEqual(bin.status_code, 302) - post_url = bin.headers['location'] + url = httpbin('post') - post2 = requests.post(post_url, files={'some': open('test_requests.py')}, - headers = {'User-Agent': 'requests-tests'}) + post2 = requests.post(url, files={'some': open('test_requests.py')}, + headers = {'User-Agent': 'requests-tests'}) - self.assertEqual(post2.status_code, 201) + self.assertEqual(post2.status_code, 200) def test_nonzero_evaluation(self): - r = requests.get('http://google.com/some-404-url') + r = requests.get(httpbin('status', '500')) self.assertEqual(bool(r), False) - r = requests.get('http://google.com/') + r = requests.get(httpbin('/')) self.assertEqual(bool(r), True) + def test_request_ok_set(self): - r = requests.get('http://google.com/some-404-url') + r = requests.get(httpbin('status', '404')) self.assertEqual(r.ok, False) + def test_status_raising(self): - r = requests.get('http://google.com/some-404-url') + r = requests.get(httpbin('status', '404')) self.assertRaises(requests.HTTPError, r.raise_for_status) - r = requests.get('http://google.com/') + r = requests.get(httpbin('status', '200')) self.assertFalse(r.error) r.raise_for_status() + def test_cookie_jar(self): - """ - .. todo:: This really doesn't test to make sure the cookie is working - """ jar = cookielib.CookieJar() self.assertFalse(jar) - requests.get('http://google.com', cookies=jar) + url = httpbin('cookies', 'set', 'requests_cookie', 'awesome') + r = requests.get(url, cookies=jar) self.assertTrue(jar) + cookie_found = False + for cookie in jar: + if cookie.name == 'requests_cookie': + self.assertEquals(cookie.value, 'awesome') + cookie_found = True + self.assertTrue(cookie_found) + + r = requests.get(httpbin('cookies'), cookies=jar) + self.assertTrue('awesome' in r.content) + + def test_decompress_gzip(self): - r = requests.get('http://api.stackoverflow.com/1.1/users/495995/top-answer-tags') + r = requests.get(httpbin('gzip')) r.content.decode('ascii') + def test_autoauth(self): - conv_auth = ('requeststest', 'requeststest') - requests.auth_manager.add_auth('convore.com', conv_auth) + http_auth = ('user', 'pass') + requests.auth_manager.add_auth('httpbin.org', http_auth) - r = requests.get('https://convore.com/api/account/verify.json') + r = requests.get(httpbin('basic-auth', 'user', 'pass')) self.assertEquals(r.status_code, 200) + def test_unicode_get(self): - requests.get('http://google.com', params={'foo': u'føø'}) - requests.get('http://google.com', params={u'føø': u'føø'}) - requests.get('http://google.com', params={'føø': 'føø'}) - requests.get('http://google.com', params={'foo': u'foo'}) - requests.get('http://google.com/ø', params={'foo': u'foo'}) + + url = httpbin('/') + + requests.get(url, params={'foo': u'føø'}) + requests.get(url, params={u'føø': u'føø'}) + requests.get(url, params={'føø': 'føø'}) + requests.get(url, params={'foo': u'foo'}) + requests.get(httpbin('ø'), params={'foo': u'foo'}) + def test_httpauth_recursion(self): - conv_auth = ('requeststest', 'bad_password') + http_auth = ('user', 'BADpass') - r = requests.get('https://convore.com/api/account/verify.json', auth=conv_auth) + r = requests.get(httpbin('basic-auth', 'user', 'pass'), auth=http_auth) self.assertEquals(r.status_code, 401) - def test_settings(self): - with requests.settings(timeout=0.0001): - self.assertRaises(requests.Timeout, requests.get, 'http://google.com') - with requests.settings(timeout=10): - requests.get('http://google.com') + def test_settings(self): + + def test(): + r = requests.get(httpbin('')) + r.raise_for_status() + + with requests.settings(timeout=0.0000001): + self.assertRaises(requests.Timeout, test) + + with requests.settings(timeout=100): + requests.get(httpbin('')) + + + def test_urlencoded_post_data(self): + r = requests.post(httpbin('post'), data=dict(test='fooaowpeuf')) + self.assertEquals(r.status_code, 200) + self.assertEquals(r.headers['content-type'], 'application/json') + self.assertEquals(r.url, httpbin('post')) + rbody = json.loads(r.content) + self.assertEquals(rbody.get('form'), dict(test='fooaowpeuf')) + self.assertEquals(rbody.get('data'), '') + def test_nonurlencoded_post_data(self): - requests.post('http://google.com', data='foo') + r = requests.post(httpbin('post'), data='fooaowpeuf') + self.assertEquals(r.status_code, 200) + self.assertEquals(r.headers['content-type'], 'application/json') + self.assertEquals(r.url, httpbin('post')) + rbody = json.loads(r.content) + # Body wasn't valid url encoded data, so the server returns None as + # "form" and the raw body as "data". + self.assertEquals(rbody.get('form'), None) + self.assertEquals(rbody.get('data'), 'fooaowpeuf') + + + def test_urlencoded_post_querystring(self): + r = requests.post(httpbin('post'), params=dict(test='fooaowpeuf')) + self.assertEquals(r.status_code, 200) + self.assertEquals(r.headers['content-type'], 'application/json') + self.assertEquals(r.url, httpbin('post?test=fooaowpeuf')) + rbody = json.loads(r.content) + self.assertEquals(rbody.get('form'), {}) # No form supplied + self.assertEquals(rbody.get('data'), '') + + + def test_nonurlencoded_post_querystring(self): + r = requests.post(httpbin('post'), params='fooaowpeuf') + self.assertEquals(r.status_code, 200) + self.assertEquals(r.headers['content-type'], 'application/json') + self.assertEquals(r.url, httpbin('post?fooaowpeuf')) + rbody = json.loads(r.content) + self.assertEquals(rbody.get('form'), {}) # No form supplied + self.assertEquals(rbody.get('data'), '') + + + def test_urlencoded_post_query_and_data(self): + r = requests.post(httpbin('post'), params=dict(test='fooaowpeuf'), + data=dict(test2="foobar")) + self.assertEquals(r.status_code, 200) + self.assertEquals(r.headers['content-type'], 'application/json') + self.assertEquals(r.url, httpbin('post?test=fooaowpeuf')) + rbody = json.loads(r.content) + self.assertEquals(rbody.get('form'), dict(test2='foobar')) + self.assertEquals(rbody.get('data'), '') + + + def test_nonurlencoded_post_query_and_data(self): + r = requests.post(httpbin('post'), params='fooaowpeuf', + data="foobar") + self.assertEquals(r.status_code, 200) + self.assertEquals(r.headers['content-type'], 'application/json') + self.assertEquals(r.url, httpbin('post?fooaowpeuf')) + rbody = json.loads(r.content) + self.assertEquals(rbody.get('form'), None) + self.assertEquals(rbody.get('data'), 'foobar') diff --git a/tox.ini b/tox.ini index 8e191e16..3c2ef1f6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,8 @@ [tox] -envlist = py25,py26,py27,pypy +envlist = py25,py26,py27 [testenv] commands=py.test --junitxml=junit-{envname}.xml deps = pytest - -[testenv:pypy] -basepython=/usr/bin/pypy - + omnijson