Merge branch 'release/0.3.4'

This commit is contained in:
Kenneth Reitz
2011-05-14 14:31:24 -04:00
8 changed files with 668 additions and 544 deletions
+1
View File
@@ -18,3 +18,4 @@ Patches and Suggestions
- Rob Madole
- Aram Dulyan
- Johannes Gorset
- 村山めがね (Megan Emurayama)
+9
View File
@@ -1,6 +1,15 @@
History
-------
0.3.4
+++++
* Urllib2 HTTPAuthentication Recursion fix (Basic/Digest)
* Internal Refactor
* Bytes data upload Bugfix
0.3.3 (2011-05-12)
++++++++++++++++++
+115
View File
@@ -0,0 +1,115 @@
# -*- coding: utf-8 -*-
"""
requests.api
~~~~~~~~~~~~
This module impliments the Requests API.
:copyright: (c) 2011 by Kenneth Reitz.
:license: ISC, see LICENSE for more details.
"""
import requests
from .models import Request, Response, AuthManager, AuthObject, auth_manager
__all__ = ('request', 'get', 'head', 'post', 'put', 'delete')
def request(method, url, **kwargs):
"""Sends a `method` request. Returns :class:`Response` object.
:param method: method for the new :class:`Request` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary of GET/HEAD/DELETE Parameters to send with the :class:`Request`.
:param data: (optional) Bytes/Dictionary of PUT/POST Data to send with the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
:param cookies: (optional) CookieJar object to send with the :class:`Request`.
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
"""
data = kwargs.pop('data', dict()) or kwargs.pop('params', dict())
r = Request(method=method, url=url, data=data, headers=kwargs.pop('headers', {}),
cookiejar=kwargs.pop('cookies', None), files=kwargs.pop('files', None),
auth=kwargs.pop('auth', auth_manager.get_auth(url)),
timeout=kwargs.pop('timeout', requests.timeout))
r.send()
return r.response
def get(url, params={}, headers={}, cookies=None, auth=None, **kwargs):
"""Sends a GET request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary of GET Parameters to send with the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
:param cookies: (optional) CookieJar object to send with the :class:`Request`.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
"""
return request('GET', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs)
def head(url, params={}, headers={}, cookies=None, auth=None, **kwargs):
"""Sends a HEAD request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary of GET Parameters to send with the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
:param cookies: (optional) CookieJar object to send with the :class:`Request`.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
"""
return request('HEAD', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs)
def post(url, data={}, headers={}, files=None, cookies=None, auth=None, **kwargs):
"""Sends a POST request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary of POST data to send with the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
:param cookies: (optional) CookieJar object to send with the :class:`Request`.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
"""
return request('POST', url, data=data, headers=headers, files=files, cookies=cookies, auth=auth, **kwargs)
def put(url, data='', headers={}, files={}, cookies=None, auth=None, **kwargs):
"""Sends a PUT request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Bytes of PUT Data to send with the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
:param cookies: (optional) CookieJar object to send with the :class:`Request`.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
"""
return request('PUT', url, data=data, headers=headers, files=files, cookies=cookies, auth=auth, **kwargs)
def delete(url, params={}, headers={}, cookies=None, auth=None, **kwargs):
"""Sends a DELETE request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary of DELETE Parameters to send with the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
:param cookies: (optional) CookieJar object to send with the :class:`Request`.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
"""
return request('DELETE', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs)
+10 -544
View File
@@ -1,557 +1,23 @@
# -*- coding: utf-8 -*-
"""
requests.core
~~~~~~~~~~~~~
requests.core
~~~~~~~~~~~~~
This module implements the main Requests system.
This module implements the main Requests system.
:copyright: (c) 2011 by Kenneth Reitz.
:license: ISC, see LICENSE for more details.
:copyright: (c) 2011 by Kenneth Reitz.
:license: ISC, see LICENSE for more details.
"""
from __future__ import absolute_import
import requests
import urllib
import urllib2
import socket
import zlib
from urllib2 import HTTPError
from urlparse import urlparse
from .packages.poster.encode import multipart_encode
from .packages.poster.streaminghttp import register_openers, get_handlers
__title__ = 'requests'
__version__ = '0.3.3'
__build__ = 0x000303
__version__ = '0.3.4'
__build__ = 0x000304
__author__ = 'Kenneth Reitz'
__license__ = 'ISC'
__copyright__ = 'Copyright 2011 Kenneth Reitz'
__all__ = [
'Request', 'Response', 'request', 'get', 'head', 'post', 'put', 'delete',
'auth_manager', 'AuthObject','RequestException', 'AuthenticationError',
'URLRequired', 'InvalidMethod', 'HTTPError'
]
class _Request(urllib2.Request):
"""Hidden wrapper around the urllib2.Request object. Allows for manual
setting of HTTP methods.
"""
def __init__(self, url, data=None, headers={}, origin_req_host=None, unverifiable=False, method=None):
urllib2.Request.__init__(self, url, data, headers, origin_req_host, unverifiable)
self.method = method
def get_method(self):
if self.method:
return self.method
return urllib2.Request.get_method(self)
class Request(object):
"""The :class:`Request` object. It carries out all functionality of
Requests. Recommended interface is with the Requests functions.
"""
_METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE')
def __init__(self, url=None, headers=dict(), files=None, method=None,
data=dict(), auth=None, cookiejar=None, timeout=None):
self.url = url
self.headers = headers
self.files = files
self.method = method
self.data = data
socket.setdefaulttimeout(timeout)
for (k, v) in self.data.iteritems():
self.data[k] = v.encode('utf-8')
# url encode data if it's a dict
if hasattr(data, 'items'):
self._enc_data = urllib.urlencode(self.data)
else:
self._enc_data = self.data
self.response = Response()
if isinstance(auth, (list, tuple)):
auth = AuthObject(*auth)
if not auth:
auth = auth_manager.get_auth(self.url)
self.auth = auth
self.cookiejar = cookiejar
self.sent = False
def __repr__(self):
return '<Request [%s]>' % (self.method)
def __setattr__(self, name, value):
if (name == 'method') and (value):
if not value in self._METHODS:
raise InvalidMethod()
object.__setattr__(self, name, value)
def _checks(self):
"""Deterministic checks for consistency."""
if not self.url:
raise URLRequired
def _get_opener(self):
"""Creates appropriate opener object for urllib2."""
_handlers = []
if self.cookiejar is not None:
_handlers.append(urllib2.HTTPCookieProcessor(self.cookiejar))
if self.auth:
if not isinstance(self.auth.handler, (urllib2.AbstractBasicAuthHandler, urllib2.AbstractDigestAuthHandler)):
auth_manager.add_password(self.auth.realm, self.url, self.auth.username, self.auth.password)
self.auth.handler = self.auth.handler(auth_manager)
auth_manager.add_auth(self.url, self.auth)
_handlers.append(self.auth.handler)
if not _handlers:
return urllib2.urlopen
_handlers.extend(get_handlers())
opener = urllib2.build_opener(*_handlers)
if self.headers:
# Allow default headers in the opener to be overloaded
normal_keys = [k.capitalize() for k in self.headers]
for key, val in opener.addheaders[:]:
if key not in normal_keys:
continue
# Remove it, we have a value to take its place
opener.addheaders.remove((key, val))
return opener.open
def _build_response(self, resp):
"""Build internal Response object from given response."""
self.response.status_code = getattr(resp, 'code', None)
self.response.headers = getattr(resp.info(), 'dict', None)
self.response.content = resp.read()
if self.response.headers.get('content-encoding', None) == 'gzip':
try:
self.response.content = zlib.decompress(self.response.content, 16+zlib.MAX_WBITS)
except zlib.error:
pass
self.response.url = getattr(resp, 'url', None)
@staticmethod
def _build_url(url, data=None):
"""Build URLs."""
if urlparse(url).query:
return '%s&%s' % (url, data)
else:
if data:
return '%s?%s' % (url, data)
else:
return url
def send(self, anyway=False):
"""Sends the request. Returns True of successful, false if not.
If there was an HTTPError during transmission,
self.response.status_code will contain the HTTPError code.
Once a request is successfully sent, `sent` will equal True.
:param anyway: If True, request will be sent, even if it has
already been sent.
"""
self._checks()
success = False
if self.method in ('GET', 'HEAD', 'DELETE'):
req = _Request(self._build_url(self.url, self._enc_data), method=self.method)
else:
if self.files:
register_openers()
if self.data:
self.files.update(self.data)
datagen, headers = multipart_encode(self.files)
req = _Request(self.url, data=datagen, headers=headers, method=self.method)
else:
req = _Request(self.url, data=self._enc_data, method=self.method)
if self.headers:
req.headers.update(self.headers)
if not self.sent or anyway:
try:
opener = self._get_opener()
resp = opener(req)
if self.cookiejar is not None:
self.cookiejar.extract_cookies(resp, req)
except urllib2.HTTPError, why:
self._build_response(why)
self.response.error = why
else:
self._build_response(resp)
self.response.ok = True
self.response.cached = False
else:
self.response.cached = True
self.sent = self.response.ok
return self.sent
def read(self, *args):
return self.response.read()
class Response(object):
"""The :class:`Request` object. All :class:`Request` objects contain a
:class:`Request.response <response>` attribute, which is an instance of
this class.
"""
def __init__(self):
self.content = None
self.status_code = None
self.headers = dict()
self.url = None
self.ok = False
self.error = None
self.cached = False
def __repr__(self):
return '<Response [%s]>' % (self.status_code)
def __nonzero__(self):
"""Returns true if status_code is 'OK'."""
return not self.error
def raise_for_status(self):
"""Raises stored HTTPError if one exists."""
if self.error:
raise self.error
def read(self, *args):
return self.content
class AuthManager(object):
"""Authentication Manager."""
def __new__(cls):
singleton = cls.__dict__.get('__singleton__')
if singleton is not None:
return singleton
cls.__singleton__ = singleton = object.__new__(cls)
return singleton
def __init__(self):
self.passwd = {}
self._auth = {}
def __repr__(self):
return '<AuthManager [%s]>' % (self.method)
def add_auth(self, uri, auth):
"""Registers AuthObject to AuthManager."""
uri = self.reduce_uri(uri, False)
# try to make it an AuthObject
if not isinstance(auth, AuthObject):
try:
auth = AuthObject(*auth)
except TypeError:
pass
self._auth[uri] = auth
def add_password(self, realm, uri, user, passwd):
"""Adds password to AuthManager."""
# uri could be a single URI or a sequence
if isinstance(uri, basestring):
uri = [uri]
reduced_uri = tuple([self.reduce_uri(u, False) for u in uri])
if reduced_uri not in self.passwd:
self.passwd[reduced_uri] = {}
self.passwd[reduced_uri] = (user, passwd)
def find_user_password(self, realm, authuri):
for uris, authinfo in self.passwd.iteritems():
reduced_authuri = self.reduce_uri(authuri, False)
for uri in uris:
if self.is_suburi(uri, reduced_authuri):
return authinfo
return (None, None)
def get_auth(self, uri):
(in_domain, in_path) = self.reduce_uri(uri, False)
for domain, path, authority in (
(i[0][0], i[0][1], i[1]) for i in self._auth.iteritems()
):
if in_domain == domain:
if path in in_path:
return authority
def reduce_uri(self, uri, default_port=True):
"""Accept authority or URI and extract only the authority and path."""
# note HTTP URLs do not have a userinfo component
parts = urllib2.urlparse.urlsplit(uri)
if parts[1]:
# URI
scheme = parts[0]
authority = parts[1]
path = parts[2] or '/'
else:
# host or host:port
scheme = None
authority = uri
path = '/'
host, port = urllib2.splitport(authority)
if default_port and port is None and scheme is not None:
dport = {"http": 80,
"https": 443,
}.get(scheme)
if dport is not None:
authority = "%s:%d" % (host, dport)
return authority, path
def is_suburi(self, base, test):
"""Check if test is below base in a URI tree
Both args must be URIs in reduced form.
"""
if base == test:
return True
if base[0] != test[0]:
return False
common = urllib2.posixpath.commonprefix((base[1], test[1]))
if len(common) == len(base[1]):
return True
return False
def empty(self):
self.passwd = {}
def remove(self, uri, realm=None):
# uri could be a single URI or a sequence
if isinstance(uri, basestring):
uri = [uri]
for default_port in True, False:
reduced_uri = tuple([self.reduce_uri(u, default_port) for u in uri])
del self.passwd[reduced_uri][realm]
def __contains__(self, uri):
# uri could be a single URI or a sequence
if isinstance(uri, basestring):
uri = [uri]
uri = tuple([self.reduce_uri(u, False) for u in uri])
if uri in self.passwd:
return True
return False
auth_manager = AuthManager()
class AuthObject(object):
"""The :class:`AuthObject` is a simple HTTP Authentication token. When
given to a Requests function, it enables Basic HTTP Authentication for that
Request. You can also enable Authorization for domain realms with AutoAuth.
See AutoAuth for more details.
:param username: Username to authenticate with.
:param password: Password for given username.
:param realm: (optional) the realm this auth applies to
:param handler: (optional) basic || digest || proxy_basic || proxy_digest
"""
_handlers = {
'basic': urllib2.HTTPBasicAuthHandler,
'digest': urllib2.HTTPDigestAuthHandler,
'proxy_basic': urllib2.ProxyBasicAuthHandler,
'proxy_digest': urllib2.ProxyDigestAuthHandler
}
def __init__(self, username, password, handler='basic', realm=None):
self.username = username
self.password = password
self.realm = realm
if isinstance(handler, basestring):
self.handler = self._handlers.get(handler.lower(), urllib2.HTTPBasicAuthHandler)
else:
self.handler = handler
def request(method, url, **kwargs):
"""Sends a `method` request. Returns :class:`Response` object.
:param method: method for the new :class:`Request` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary of GET/HEAD/DELETE Parameters to send with the :class:`Request`.
:param data: (optional) Bytes/Dictionary of PUT/POST Data to send with the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
:param cookies: (optional) CookieJar object to send with the :class:`Request`.
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
"""
data = kwargs.pop('data', dict()) or kwargs.pop('params', dict())
r = Request(method=method, url=url, data=data, headers=kwargs.pop('headers', {}),
cookiejar=kwargs.pop('cookies', None), files=kwargs.pop('files', None),
auth=kwargs.pop('auth', auth_manager.get_auth(url)),
timeout=kwargs.pop('timeout', requests.timeout))
r.send()
return r.response
def get(url, params={}, headers={}, cookies=None, auth=None, **kwargs):
"""Sends a GET request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary of GET Parameters to send with the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
:param cookies: (optional) CookieJar object to send with the :class:`Request`.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
"""
return request('GET', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs)
def head(url, params={}, headers={}, cookies=None, auth=None, **kwargs):
"""Sends a HEAD request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary of GET Parameters to send with the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
:param cookies: (optional) CookieJar object to send with the :class:`Request`.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
"""
return request('HEAD', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs)
def post(url, data={}, headers={}, files=None, cookies=None, auth=None, **kwargs):
"""Sends a POST request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param data: (optional) Dictionary of POST data to send with the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
:param cookies: (optional) CookieJar object to send with the :class:`Request`.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
"""
return request('POST', url, data=data, headers=headers, files=files, cookies=cookies, auth=auth, **kwargs)
def put(url, data='', headers={}, files={}, cookies=None, auth=None, **kwargs):
"""Sends a PUT request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Bytes of PUT Data to send with the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
:param cookies: (optional) CookieJar object to send with the :class:`Request`.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
"""
return request('PUT', url, data=data, headers=headers, files=files, cookies=cookies, auth=auth, **kwargs)
def delete(url, params={}, headers={}, cookies=None, auth=None, **kwargs):
"""Sends a DELETE request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object.
:param params: (optional) Dictionary of DELETE Parameters to send with the :class:`Request`.
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
:param cookies: (optional) CookieJar object to send with the :class:`Request`.
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
:param timeout: (optional) Float describing the timeout of the request.
"""
return request('DELETE', url, params=params, headers=headers, cookies=cookies, auth=auth, **kwargs)
class RequestException(Exception):
"""There was an ambiguous exception that occured while handling your
request."""
class AuthenticationError(RequestException):
"""The authentication credentials provided were invalid."""
class URLRequired(RequestException):
"""A valid URL is required to make a request."""
class InvalidMethod(RequestException):
"""An inappropriate method was attempted."""
from .models import HTTPError, auth_manager
from .api import *
+435
View File
@@ -0,0 +1,435 @@
# -*- coding: utf-8 -*-
"""
requests.system
~~~~~~~~~~~~~~~
"""
import requests
import urllib
import urllib2
import socket
import zlib
from urllib2 import HTTPError
from urlparse import urlparse
from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPDigestAuthHandler
from .packages.poster.encode import multipart_encode
from .packages.poster.streaminghttp import register_openers, get_handlers
class Request(object):
"""The :class:`Request` object. It carries out all functionality of
Requests. Recommended interface is with the Requests functions.
"""
_METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE')
def __init__(self, url=None, headers=dict(), files=None, method=None,
data=dict(), auth=None, cookiejar=None, timeout=None):
socket.setdefaulttimeout(timeout)
self.url = url
self.headers = headers
self.files = files
self.method = method
self.data = {}
# self.data = {}
if hasattr(data, 'items'):
for (k, v) in data.items():
self.data.update({
k.encode('utf-8') if isinstance(k, unicode) else k:
v.encode('utf-8') if isinstance(v, unicode) else v
})
# url encode data if it's a dict
if hasattr(data, 'items'):
self._enc_data = urllib.urlencode(self.data)
else:
self._enc_data = data
self.response = Response()
if isinstance(auth, (list, tuple)):
auth = AuthObject(*auth)
if not auth:
auth = auth_manager.get_auth(self.url)
self.auth = auth
self.cookiejar = cookiejar
self.sent = False
def __repr__(self):
return '<Request [%s]>' % (self.method)
def __setattr__(self, name, value):
if (name == 'method') and (value):
if not value in self._METHODS:
raise InvalidMethod()
object.__setattr__(self, name, value)
def _checks(self):
"""Deterministic checks for consistency."""
if not self.url:
raise URLRequired
def _get_opener(self):
"""Creates appropriate opener object for urllib2."""
_handlers = []
if self.cookiejar is not None:
_handlers.append(urllib2.HTTPCookieProcessor(self.cookiejar))
if self.auth:
if not isinstance(self.auth.handler, (urllib2.AbstractBasicAuthHandler, urllib2.AbstractDigestAuthHandler)):
auth_manager.add_password(self.auth.realm, self.url, self.auth.username, self.auth.password)
self.auth.handler = self.auth.handler(auth_manager)
auth_manager.add_auth(self.url, self.auth)
_handlers.append(self.auth.handler)
if not _handlers:
return urllib2.urlopen
_handlers.extend(get_handlers())
opener = urllib2.build_opener(*_handlers)
if self.headers:
# Allow default headers in the opener to be overloaded
normal_keys = [k.capitalize() for k in self.headers]
for key, val in opener.addheaders[:]:
if key not in normal_keys:
continue
# Remove it, we have a value to take its place
opener.addheaders.remove((key, val))
return opener.open
def _build_response(self, resp):
"""Build internal Response object from given response."""
if isinstance(resp, HTTPError):
# print resp.__dict__
pass
self.response.status_code = getattr(resp, 'code', None)
try:
self.response.headers = getattr(resp.info(), 'dict', None)
self.response.content = resp.read()
except AttributeError, why:
pass
if self.response.headers.get('content-encoding', None) == 'gzip':
try:
self.response.content = zlib.decompress(self.response.content, 16+zlib.MAX_WBITS)
except zlib.error:
pass
self.response.url = getattr(resp, 'url', None)
@staticmethod
def _build_url(url, data=None):
"""Build URLs."""
if urlparse(url).query:
return '%s&%s' % (url, data)
else:
if data:
return '%s?%s' % (url, data)
else:
return url
def send(self, anyway=False):
"""Sends the request. Returns True of successful, false if not.
If there was an HTTPError during transmission,
self.response.status_code will contain the HTTPError code.
Once a request is successfully sent, `sent` will equal True.
:param anyway: If True, request will be sent, even if it has
already been sent.
"""
self._checks()
success = False
if self.method in ('GET', 'HEAD', 'DELETE'):
req = _Request(self._build_url(self.url, self._enc_data), method=self.method)
else:
if self.files:
register_openers()
if self.data:
self.files.update(self.data)
datagen, headers = multipart_encode(self.files)
req = _Request(self.url, data=datagen, headers=headers, method=self.method)
else:
req = _Request(self.url, data=self._enc_data, method=self.method)
if self.headers:
req.headers.update(self.headers)
if not self.sent or anyway:
try:
opener = self._get_opener()
resp = opener(req)
if self.cookiejar is not None:
self.cookiejar.extract_cookies(resp, req)
except urllib2.HTTPError, why:
self._build_response(why)
self.response.error = why
else:
self._build_response(resp)
self.response.ok = True
self.response.cached = False
else:
self.response.cached = True
self.sent = self.response.ok
return self.sent
def read(self, *args):
return self.response.read()
class Response(object):
"""The :class:`Request` object. All :class:`Request` objects contain a
:class:`Request.response <response>` attribute, which is an instance of
this class.
"""
def __init__(self):
self.content = None
self.status_code = None
self.headers = dict()
self.url = None
self.ok = False
self.error = None
self.cached = False
def __repr__(self):
return '<Response [%s]>' % (self.status_code)
def __nonzero__(self):
"""Returns true if status_code is 'OK'."""
return not self.error
def raise_for_status(self):
"""Raises stored HTTPError if one exists."""
if self.error:
raise self.error
def read(self, *args):
return self.content
class AuthManager(object):
"""Authentication Manager."""
def __new__(cls):
singleton = cls.__dict__.get('__singleton__')
if singleton is not None:
return singleton
cls.__singleton__ = singleton = object.__new__(cls)
return singleton
def __init__(self):
self.passwd = {}
self._auth = {}
def __repr__(self):
return '<AuthManager [%s]>' % (self.method)
def add_auth(self, uri, auth):
"""Registers AuthObject to AuthManager."""
uri = self.reduce_uri(uri, False)
# try to make it an AuthObject
if not isinstance(auth, AuthObject):
try:
auth = AuthObject(*auth)
except TypeError:
pass
self._auth[uri] = auth
def add_password(self, realm, uri, user, passwd):
"""Adds password to AuthManager."""
# uri could be a single URI or a sequence
if isinstance(uri, basestring):
uri = [uri]
reduced_uri = tuple([self.reduce_uri(u, False) for u in uri])
if reduced_uri not in self.passwd:
self.passwd[reduced_uri] = {}
self.passwd[reduced_uri] = (user, passwd)
def find_user_password(self, realm, authuri):
for uris, authinfo in self.passwd.iteritems():
reduced_authuri = self.reduce_uri(authuri, False)
for uri in uris:
if self.is_suburi(uri, reduced_authuri):
return authinfo
return (None, None)
def get_auth(self, uri):
(in_domain, in_path) = self.reduce_uri(uri, False)
for domain, path, authority in (
(i[0][0], i[0][1], i[1]) for i in self._auth.iteritems()
):
if in_domain == domain:
if path in in_path:
return authority
def reduce_uri(self, uri, default_port=True):
"""Accept authority or URI and extract only the authority and path."""
# note HTTP URLs do not have a userinfo component
parts = urllib2.urlparse.urlsplit(uri)
if parts[1]:
# URI
scheme = parts[0]
authority = parts[1]
path = parts[2] or '/'
else:
# host or host:port
scheme = None
authority = uri
path = '/'
host, port = urllib2.splitport(authority)
if default_port and port is None and scheme is not None:
dport = {"http": 80,
"https": 443,
}.get(scheme)
if dport is not None:
authority = "%s:%d" % (host, dport)
return authority, path
def is_suburi(self, base, test):
"""Check if test is below base in a URI tree
Both args must be URIs in reduced form.
"""
if base == test:
return True
if base[0] != test[0]:
return False
common = urllib2.posixpath.commonprefix((base[1], test[1]))
if len(common) == len(base[1]):
return True
return False
def empty(self):
self.passwd = {}
def remove(self, uri, realm=None):
# uri could be a single URI or a sequence
if isinstance(uri, basestring):
uri = [uri]
for default_port in True, False:
reduced_uri = tuple([self.reduce_uri(u, default_port) for u in uri])
del self.passwd[reduced_uri][realm]
def __contains__(self, uri):
# uri could be a single URI or a sequence
if isinstance(uri, basestring):
uri = [uri]
uri = tuple([self.reduce_uri(u, False) for u in uri])
if uri in self.passwd:
return True
return False
auth_manager = AuthManager()
class AuthObject(object):
"""The :class:`AuthObject` is a simple HTTP Authentication token. When
given to a Requests function, it enables Basic HTTP Authentication for that
Request. You can also enable Authorization for domain realms with AutoAuth.
See AutoAuth for more details.
:param username: Username to authenticate with.
:param password: Password for given username.
:param realm: (optional) the realm this auth applies to
:param handler: (optional) basic || digest || proxy_basic || proxy_digest
"""
_handlers = {
'basic': HTTPBasicAuthHandler,
'digest': HTTPDigestAuthHandler,
'proxy_basic': urllib2.ProxyBasicAuthHandler,
'proxy_digest': urllib2.ProxyDigestAuthHandler
}
def __init__(self, username, password, handler='basic', realm=None):
self.username = username
self.password = password
self.realm = realm
if isinstance(handler, basestring):
self.handler = self._handlers.get(handler.lower(), urllib2.HTTPBasicAuthHandler)
else:
self.handler = handler
class RequestException(Exception):
"""There was an ambiguous exception that occured while handling your
request."""
class AuthenticationError(RequestException):
"""The authentication credentials provided were invalid."""
class URLRequired(RequestException):
"""A valid URL is required to make a request."""
class InvalidMethod(RequestException):
"""An inappropriate method was attempted."""
+81
View File
@@ -0,0 +1,81 @@
# -*- coding: utf-8 -*-
"""
requests.monkeys
~~~~~~~~~~~~~~~~
Urllib2 Monkey patches.
"""
import urllib2
class Request(urllib2.Request):
"""Hidden wrapper around the urllib2.Request object. Allows for manual
setting of HTTP methods.
"""
def __init__(self, url, data=None, headers={}, origin_req_host=None, unverifiable=False, method=None):
urllib2.Request.__init__(self, url, data, headers, origin_req_host, unverifiable)
self.method = method
def get_method(self):
if self.method:
return self.method
return urllib2.Request.get_method(self)
class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler):
# from mercurial
def __init__(self, *args, **kwargs):
urllib2.HTTPBasicAuthHandler.__init__(self, *args, **kwargs)
self.retried_req = None
def reset_retry_count(self):
# Python 2.6.5 will call this on 401 or 407 errors and thus loop
# forever. We disable reset_retry_count completely and reset in
# http_error_auth_reqed instead.
pass
def http_error_auth_reqed(self, auth_header, host, req, headers):
# Reset the retry counter once for each request.
if req is not self.retried_req:
self.retried_req = req
self.retried = 0
return urllib2.HTTPBasicAuthHandler.http_error_auth_reqed(
self, auth_header, host, req, headers)
class HTTPDigestAuthHandler(urllib2.HTTPDigestAuthHandler):
def __init__(self, *args, **kwargs):
urllib2.HTTPDigestAuthHandler.__init__(self, *args, **kwargs)
self.retried_req = None
def reset_retry_count(self):
# Python 2.6.5 will call this on 401 or 407 errors and thus loop
# forever. We disable reset_retry_count completely and reset in
# http_error_auth_reqed instead.
pass
def http_error_auth_reqed(self, auth_header, host, req, headers):
# Reset the retry counter once for each request.
if req is not self.retried_req:
self.retried_req = req
self.retried = 0
# In python < 2.5 AbstractDigestAuthHandler raises a ValueError if
# it doesn't know about the auth type requested. This can happen if
# somebody is using BasicAuth and types a bad password.
try:
return urllib2.HTTPDigestAuthHandler.http_error_auth_reqed(
self, auth_header, host, req, headers)
except ValueError, inst:
arg = inst.args[0]
if arg.startswith("AbstractDigestAuthHandler doesn't know "):
return
raise
+5
View File
@@ -0,0 +1,5 @@
# -*- coding: utf-8 -*-
"""
requests.monkeys
"""
+12
View File
@@ -79,6 +79,9 @@ class RequestsTestSuite(unittest.TestCase):
post2 = requests.post(bin.url, files={'some': open('test_requests.py')})
self.assertEqual(post2.status_code, 201)
post3 = requests.post(bin.url, data='[{"some": "json"}]')
self.assertEqual(post.status_code, 201)
def test_POSTBIN_GET_POST_FILES_WITH_PARAMS(self):
bin = requests.post('http://www.postbin.org/')
@@ -120,6 +123,7 @@ class RequestsTestSuite(unittest.TestCase):
"""
.. todo:: This really doesn't test to make sure the cookie is working
"""
jar = cookielib.CookieJar()
self.assertFalse(jar)
@@ -141,9 +145,17 @@ class RequestsTestSuite(unittest.TestCase):
def test_unicode_get(self):
requests.get('http://google.com', params={'foo': u'føø'})
requests.get('http://google.com', params={u'føø': u'føø'})
requests.get('http://google.com', params={'føø': 'føø'})
requests.get('http://google.com', params={'foo': u'foo'})
requests.get('http://google.com/ø', params={'foo': u'foo'})
def test_httpauth_recursion(self):
conv_auth = ('requeststest', 'bad_password')
r = requests.get('https://convore.com/api/account/verify.json', auth=conv_auth)
self.assertEquals(r.status_code, 401)
if __name__ == '__main__':
unittest.main()