kennethreitz/requests
1
0
Fork 0
mirror of https://github.com/kennethreitz/requests.git synced 2026-06-05 22:50:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' into urllib3

Browse Source 
Conflicts:
	requests/api.py
	requests/exceptions.py
	requests/models.py
This commit is contained in:
Kenneth Reitz
2011-09-25 16:00:44 -04:00
parent 04bcab8057 cd01aa4666
commit d2894b3c58
12 changed files with 108 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AUTHORS
+2 -1
View File
@@ -49,5 +49,6 @@ Patches and Suggestions
- Rick Mak
- Johan Bergström
- Josselin Jacquard
- Michael Van Veen <michael@mvanveen.net>
- Mike Waldner
- Serge Domkowski
- Serge Domkowski
docs/index.rst
+2 -1
View File
@@ -41,7 +41,8 @@ Testimonials
`Twitter, Inc <http://twitter.com>`_,
a U.S. Federal Institution,
and `Readability <http://readability.com>`_
`Readability <http://readability.com>`_, and
`Work for Pie <http://workforpie.com>`_
use Requests internally.
**Daniel Greenfeld**
requests/api.py
+22
View File
@@ -56,6 +56,17 @@ def request(method, url,
headers[k] = header_expand(v)
args = dict(
method = method,
url = url,
data = data,
params = params,
headers = headers,
cookiejar = cookies,
files = files,
auth = auth,
timeout = timeout or config.settings.timeout,
allow_redirects = allow_redirects,
proxies = proxies or config.settings.proxies,
method = method,
url = url,
data = data,
@@ -67,6 +78,17 @@ def request(method, url,
timeout = timeout or config.settings.timeout,
allow_redirects = allow_redirects,
proxies = proxies or config.settings.proxies
method=method,
url=url,
data=data,
params=params,
headers=headers,
cookiejar=cookies,
files=files,
auth=auth,
timeout=timeout or config.settings.timeout,
allow_redirects=allow_redirects,
proxies=proxies or config.settings.proxies,
)
# Arguments manipulation hook.
requests/config.py
+15
View File
@@ -6,8 +6,23 @@ requests.config
This module provides the Requests settings feature set.
settings parameters:
TODO: Verify!!!
TODO: Make sure format is acceptabl/cool
- :base_headers: - Sets default User-Agent to `python-requests.org`
- :accept_gzip: - Whether or not to accept gzip-compressed data
- :proxies: - http proxies?
- :verbose: - display verbose information?
- :timeout: - timeout time until request terminates
- :max_redirects: - maximum number of allowed redirects?
- :decode_unicode: - whether or not to accept unicode?
Used globally
"""
class Settings(object):
def __init__(self, **kwargs):
requests/core.py
+3 -3
View File
@@ -19,11 +19,11 @@ __license__ = 'ISC'
__copyright__ = 'Copyright 2011 Kenneth Reitz'
from models import HTTPError, Request, Response
from api import *
from config import settings
from exceptions import *
from models import HTTPError, Request, Response
from sessions import session
from status_codes import codes
from config import settings
import utils
import utils
requests/exceptions.py
+18
View File
@@ -6,15 +6,33 @@ requests.exceptions
"""
class RequestException(Exception):
"""There was an ambiguous exception that occured while handling your
request."""
class AuthenticationError(RequestException):
"""The authentication credentials provided were invalid."""
class AuthenticationError(RequestException):
"""The authentication credentials provided were invalid."""
class Timeout(RequestException):
"""The request timed out."""
class URLRequired(RequestException):
"""A valid URL is required to make a request."""
class InvalidMethod(RequestException):
"""An inappropriate method was attempted."""
class InvalidMethod(RequestException):
"""An inappropriate method was attempted."""
class TooManyRedirects(RequestException):
"""Too many redirects."""
requests/models.py
+24 -39
View File
@@ -30,7 +30,6 @@ from .exceptions import RequestException, Timeout, URLRequired, TooManyRedirects
REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
class Request(object):
"""The :class:`Request <Request>` object. It carries out all functionality of
Requests. Recommended interface is with the Requests functions.
@@ -96,7 +95,6 @@ class Request(object):
#: True if Request has been sent.
self.sent = False
# Header manipulation and defaults.
if settings.accept_gzip:
@@ -113,18 +111,15 @@ class Request(object):
self.headers = headers
def __repr__(self):
return '<Request [%s]>' % (self.method)
def _checks(self):
"""Deterministic checks for consistency."""
if not self.url:
raise URLRequired
def _get_opener(self):
"""Creates appropriate opener object for urllib2."""
@@ -173,13 +168,11 @@ class Request(object):
return opener.open
def _build_response(self, resp, is_error=False):
"""Build internal :class:`Response <Response>` object
from given response.
"""
def build(resp):
response = Response()
@@ -191,8 +184,13 @@ class Request(object):
# if self.cookiejar:
response.cookies = dict_from_cookiejar(self.cookiejar)
# response.cookies = dict_from_cookiejar(self.cookiejar)
response.cookies = dict_from_cookiejar(self.cookiejar)
except AttributeError:
pass
@@ -201,7 +199,6 @@ class Request(object):
return response
# Request collector.
history = []
@@ -280,7 +277,6 @@ class Request(object):
# Give Response some context.
self.response.request = self
@staticmethod
def _encode_params(data):
"""Encode parameters in a piece of data.
@@ -298,12 +294,13 @@ class Request(object):
for k, vs in data.items():
for v in isinstance(vs, list) and vs or [vs]:
result.append((k.encode('utf-8') if isinstance(k, unicode) else k,
v.encode('utf-8') if isinstance(v, unicode) else v))
return result, urllib.urlencode(result, doseq=True)
v.encode('utf-8') if isinstance(v, unicode) else v)
)
return (result, urllib.urlencode(result, doseq=True))
else:
return data, data
def _build_url(self):
"""Build the actual URL to use."""
@@ -322,6 +319,9 @@ class Request(object):
# Turn it back into a bytestring.
self.url = str(urlunparse([scheme, netloc, path, params, query, fragment]))
self.url = str(urlunparse(
[scheme, netloc, path, params, query, fragment]
))
# Query Parameters?
if self._enc_params:
@@ -338,7 +338,6 @@ class Request(object):
# Kosher URL.
return self.url
def send(self, connection=None, anyway=False):
"""Sends the shit."""
@@ -432,7 +431,6 @@ class Request(object):
datetime.now().isoformat(), self.method, self.url
))
url = self._build_url()
if self.method in ('GET', 'HEAD', 'DELETE'):
req = _Request(url, method=self.method)
@@ -451,7 +449,7 @@ class Request(object):
req = _Request(url, data=self._enc_data, method=self.method)
if self.headers:
for k,v in self.headers.iteritems():
for k, v in self.headers.iteritems():
req.add_header(k, v)
if not self.sent or anyway:
@@ -492,17 +490,17 @@ class Request(object):
self._build_response(resp)
self.response.ok = True
self.sent = self.response.ok
return self.sent
class Response(object):
"""The core :class:`Response <Response>` object. All
:class:`Request <Request>` objects contain a
:class:`response <Response>` attribute, which is an instance
of this class.
"""The core :class:`Response <Response>` object.
All :class:`Request <Request>` objects contain a :class:`response
<Response>` attribute, which is an instance of this class.
"""
def __init__(self):
@@ -538,11 +536,9 @@ class Response(object):
#: A dictionary of Cookies the server sent back.
self.cookies = None
def __repr__(self):
return '<Response [%s]>' % (self.status_code)
def __nonzero__(self):
"""Returns true if :attr:`status_code` is 'OK'."""
@@ -607,7 +603,6 @@ class Response(object):
self._content_consumed = True
return self._content
def raise_for_status(self):
"""Raises stored :class:`HTTPError` or :class:`URLError`,
if one occured.
@@ -619,6 +614,7 @@ class Response(object):
if (self.status_code >= 300) and (self.status_code < 400):
raise Exception('300 yo')
elif (self.status_code >= 400) and (self.status_code < 500):
raise Exception('400 yo')
@@ -638,16 +634,13 @@ class AuthManager(object):
return singleton
def __init__(self):
self.passwd = {}
self._auth = {}
def __repr__(self):
return '<AuthManager [%s]>' % (self.method)
def add_auth(self, uri, auth):
"""Registers AuthObject to AuthManager."""
@@ -662,7 +655,6 @@ class AuthManager(object):
self._auth[uri] = auth
def add_password(self, realm, uri, user, passwd):
"""Adds password to AuthManager."""
# uri could be a single URI or a sequence
@@ -675,7 +667,6 @@ class AuthManager(object):
self.passwd[reduced_uri] = {}
self.passwd[reduced_uri] = (user, passwd)
def find_user_password(self, realm, authuri):
for uris, authinfo in self.passwd.iteritems():
reduced_authuri = self.reduce_uri(authuri, False)
@@ -685,7 +676,6 @@ class AuthManager(object):
return (None, None)
def get_auth(self, uri):
(in_domain, in_path) = self.reduce_uri(uri, False)
@@ -696,7 +686,6 @@ class AuthManager(object):
if path in in_path:
return authority
def reduce_uri(self, uri, default_port=True):
"""Accept authority or URI and extract only the authority and path."""
@@ -725,7 +714,6 @@ class AuthManager(object):
return authority, path
def is_suburi(self, base, test):
"""Check if test is below base in a URI tree
@@ -740,11 +728,9 @@ class AuthManager(object):
return True
return False
def empty(self):
self.passwd = {}
def remove(self, uri, realm=None):
# uri could be a single URI or a sequence
if isinstance(uri, basestring):
@@ -754,7 +740,6 @@ class AuthManager(object):
reduced_uri = tuple([self.reduce_uri(u, default_port) for u in uri])
del self.passwd[reduced_uri][realm]
def __contains__(self, uri):
# uri could be a single URI or a sequence
if isinstance(uri, basestring):
@@ -770,12 +755,12 @@ class AuthManager(object):
auth_manager = AuthManager()
class AuthObject(object):
"""The :class:`AuthObject` is a simple HTTP Authentication token. When
given to a Requests function, it enables Basic HTTP Authentication for that
Request. You can also enable Authorization for domain realms with AutoAuth.
See AutoAuth for more details.
"""The :class:`AuthObject` is a simple HTTP Authentication token.
When given to a Requests function, it enables Basic HTTP Authentication
for that Request. You can also enable Authorization for domain realms
with AutoAuth. See AutoAuth for more details.
:param username: Username to authenticate with.
:param password: Password for given username.
requests/monkeys.py
+4 -10
View File
@@ -8,8 +8,9 @@ Urllib2 Monkey patches.
"""
import urllib2
import re
import urllib2
class Request(urllib2.Request):
"""Hidden wrapper around the urllib2.Request object. Allows for manual
@@ -35,7 +36,6 @@ class HTTPRedirectHandler(urllib2.HTTPRedirectHandler):
http_error_302 = http_error_303 = http_error_307 = http_error_301
class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler):
"""HTTP Basic Auth Handler with authentication loop fixes."""
@@ -44,14 +44,12 @@ class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler):
self.retried_req = None
self.retried = 0
def reset_retry_count(self):
# Python 2.6.5 will call this on 401 or 407 errors and thus loop
# forever. We disable reset_retry_count completely and reset in
# http_error_auth_reqed instead.
pass
def http_error_auth_reqed(self, auth_header, host, req, headers):
# Reset the retry counter once for each request.
if req is not self.retried_req:
@@ -63,7 +61,6 @@ class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler):
)
class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler):
"""HTTP Basic Auth Handler with forced Authentication."""
@@ -71,10 +68,9 @@ class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler):
rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+'
'realm=(["\'])(.*?)\\2', re.I)
def __init__(self, *args, **kwargs):
def __init__(self, *args, **kwargs):
HTTPBasicAuthHandler.__init__(self, *args, **kwargs)
def http_error_401(self, req, fp, code, msg, headers):
url = req.get_full_url()
response = self._http_error_auth_reqed('www-authenticate', url, req, headers)
@@ -83,7 +79,6 @@ class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler):
http_error_404 = http_error_401
def _http_error_auth_reqed(self, authreq, host, req, headers):
authreq = headers.get(authreq, None)
@@ -116,7 +111,6 @@ class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler):
return response
class HTTPDigestAuthHandler(urllib2.HTTPDigestAuthHandler):
def __init__(self, *args, **kwargs):
@@ -145,4 +139,4 @@ class HTTPDigestAuthHandler(urllib2.HTTPDigestAuthHandler):
arg = inst.args[0]
if arg.startswith("AbstractDigestAuthHandler doesn't know "):
return
raise
raise
requests/sessions.py
+1 -5
View File
@@ -15,13 +15,11 @@ from . import api
from .utils import add_dict_to_cookiejar
class Session(object):
"""A Requests session."""
__attrs__ = ['headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks']
def __init__(self, **kwargs):
# Set up a CookieJar to be used by default
@@ -34,7 +32,6 @@ class Session(object):
# Map and wrap requests.api methods
self._map_api_methods()
def __repr__(self):
return '<requests-client at 0x%x>' % (id(self))
@@ -45,7 +42,6 @@ class Session(object):
# print args
pass
def _map_api_methods(self):
"""Reads each available method from requests.api and decorates
them with a wrapper, which inserts any instance-local attributes
@@ -81,4 +77,4 @@ class Session(object):
def session(**kwargs):
"""Returns a :class:`Session` for context-managment."""
return Session(**kwargs)
return Session(**kwargs)
requests/status_codes.py
+1 -1
View File
@@ -80,4 +80,4 @@ for (code, titles) in _codes.items():
for title in titles:
setattr(codes, title, code)
if not title.startswith('\\'):
setattr(codes, title.upper(), code)
setattr(codes, title.upper(), code)
requests/structures.py
+3 -1
View File
@@ -8,6 +8,7 @@ Datastructures that power Requests.
"""
class CaseInsensitiveDict(dict):
"""Case-insensitive Dictionary
@@ -46,6 +47,7 @@ class CaseInsensitiveDict(dict):
else:
return default
class LookupDict(dict):
"""Dictionary lookup object."""
@@ -62,4 +64,4 @@ class LookupDict(dict):
return self.__dict__.get(key, None)
def get(self, key, default=None):
return self.__dict__.get(key, default)
return self.__dict__.get(key, default)
requests/utils.py
+13 -10
View File
@@ -51,10 +51,9 @@ def header_expand(headers):
collector.append('; '.join(_params))
if not len(headers) == i+1:
if not len(headers) == i + 1:
collector.append(', ')
# Remove trailing seperators.
if collector[-1] in (', ', '; '):
del collector[-1]
@@ -62,7 +61,6 @@ def header_expand(headers):
return ''.join(collector)
def dict_from_cookiejar(cj):
"""Returns a key/value dictionary from a CookieJar.
@@ -181,12 +179,12 @@ def unicode_from_html(content):
def stream_decode_response_unicode(iterator, r):
"""Stream decodes a iterator."""
encoding = get_encoding_from_headers(r.headers)
if encoding is None:
try:
decoder = codecs.getincrementaldecoder(str(encoding))(errors='replace')
except LookupError:
for item in iterator:
yield item
return
decoder = codecs.getincrementaldecoder(encoding)(errors='replace')
for chunk in iterator:
rv = decoder.decode(chunk)
if rv:
@@ -221,6 +219,8 @@ def get_unicode_from_response(r):
return unicode(r.content, encoding)
except UnicodeError:
tried_encodings.append(encoding)
except LookupError:
return r.content
# Fall back:
try:
@@ -235,8 +235,8 @@ def decode_gzip(content):
:param content: bytestring to gzip-decode.
"""
return zlib.decompress(content, 16+zlib.MAX_WBITS)
return zlib.decompress(content, 16+zlib.MAX_WBITS)
return zlib.decompress(content, 16 + zlib.MAX_WBITS)
return zlib.decompress(content, 16 + zlib.MAX_WBITS)
return zlib.decompress(content, 16 + zlib.MAX_WBITS)
@@ -255,6 +255,7 @@ def stream_decode_gzip(iterator):
except zlib.error:
pass
def curl_from_request(request):
"""Returns a curl command from the request.
@@ -276,10 +277,13 @@ def curl_from_request(request):
#: -u/--user - Specify the user name and password to use for server auth.
#: Basic Auth only for now
auth = ''
if request.auth is not None:
auth = '-u "%s:%s" ' % (request.auth.username, request.auth.password)
auth = '-u "%s:%s" ' % (request.auth.username, request.auth.password)
method = ''
if request.method.upper() == 'HEAD':
#: -I/--head - fetch headers only.
method = '-I '
@@ -321,4 +325,3 @@ def curl_from_request(request):
#: Params handled in _build_url
return curl + auth + method + header + cookies + form + '"' + request._build_url() + '"'