mirror of
https://github.com/kennethreitz/requests.git
synced 2026-06-05 22:50:18 +00:00
Merge pull request #202 from kennethreitz/feature/content-stream
Content Streaming
This commit is contained in:
Kenneth Reitz
@@ -40,3 +40,7 @@ Patches and Suggestions
|
||||
- Mikko Ohtamaa
|
||||
- Den Shabalin
|
||||
- Daniel Miller <danielm@vs-networks.com>
|
||||
- Alejandro Giacometti
|
||||
- Rick Mak
|
||||
- Johan Bergström
|
||||
- Josselin Jacquard
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
Requests is written and maintained by Kenneth Reitz and
|
||||
various contributors:
|
||||
|
||||
Development Lead
|
||||
````````````````
|
||||
|
||||
- Kenneth Reitz <me@kennethreitz.com>
|
||||
|
||||
|
||||
Patches and Suggestions
|
||||
```````````````````````
|
||||
|
||||
- Various Pocoo Members
|
||||
- Chris Adams
|
||||
- Flavio Percoco Premoli
|
||||
- Dj Gilcrease
|
||||
- Justin Murphy
|
||||
- Rob Madole
|
||||
- Aram Dulyan
|
||||
- Johannes Gorset
|
||||
- 村山めがね (Megane Murayama)
|
||||
- James Rowe
|
||||
- Daniel Schauenberg
|
||||
- Zbigniew Siciarz
|
||||
- Daniele Tricoli 'Eriol'
|
||||
- Richard Boulton
|
||||
- Miguel Olivares <miguel@moliware.com>
|
||||
- Alberto Paro
|
||||
- Jérémy Bethmont
|
||||
- 潘旭 (Xu Pan)
|
||||
- Tamás Gulácsi
|
||||
- Rubén Abad
|
||||
- Peter Manser
|
||||
- Jeremy Selie
|
||||
<<<<<<< HEAD
|
||||
- Jens Diemer
|
||||
- Alex <@alopatin>
|
||||
=======
|
||||
- Tom Hogans <tomhsx@gmail.com>
|
||||
>>>>>>> 0ed641a26ec2200de00e4bbf3d170c767375351e
|
||||
+9
-1
@@ -1,12 +1,20 @@
|
||||
History
|
||||
-------
|
||||
|
||||
0.6.4 (2011-10-13)
|
||||
++++++++++++++++++
|
||||
|
||||
* Automatic decoding of unicode, based on HTTP Headers.
|
||||
* New ``decode_unicode`` setting
|
||||
* Removal of ``r.read/close`` methods
|
||||
* New ``r.faw`` interface for advanced response usage.*
|
||||
* Automatic expansion of parameterized headers
|
||||
|
||||
0.6.3 (2011-10-13)
|
||||
++++++++++++++++++
|
||||
|
||||
* Beautiful ``requests.async`` module, for making async requests w/ gevent.
|
||||
|
||||
|
||||
0.6.2 (2011-10-09)
|
||||
++++++++++++++++++
|
||||
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
include README.rst LICENSE HISTORY.rst
|
||||
include README.rst LICENSE HISTORY.rst test_requests.py
|
||||
|
||||
+15
-3
@@ -31,6 +31,21 @@ They all return an instance of the :class:`Response <Response>` object.
|
||||
.. autoclass:: Response
|
||||
:inherited-members:
|
||||
|
||||
Async
|
||||
-----
|
||||
|
||||
.. module:: requests.async
|
||||
|
||||
|
||||
.. autofunction:: map
|
||||
.. autofunction:: request
|
||||
.. autofunction:: head
|
||||
.. autofunction:: get
|
||||
.. autofunction:: post
|
||||
.. autofunction:: put
|
||||
.. autofunction:: patch
|
||||
.. autofunction:: delete
|
||||
|
||||
|
||||
|
||||
Utilities
|
||||
@@ -48,9 +63,6 @@ Cookies
|
||||
.. autofunction:: cookiejar_from_dict
|
||||
.. autofunction:: add_dict_to_cookiejar
|
||||
|
||||
Curl
|
||||
~~~~
|
||||
.. autofunction:: curl_from_request
|
||||
|
||||
Encodings
|
||||
~~~~~~~~~
|
||||
|
||||
@@ -10,6 +10,7 @@ Encoded Data?
|
||||
|
||||
Requests automatically decompresses gzip-encoded responses, and does
|
||||
its best to decodes response content to unicode when possible.
|
||||
it's best to decodes response content to unicode when possible.
|
||||
|
||||
You can get direct access to the raw response (and even the socket),
|
||||
if needed as well.
|
||||
|
||||
@@ -75,4 +75,4 @@ Ubuntu::
|
||||
|
||||
Once you have ``libevent``, you can install ``gevent`` with ``pip``::
|
||||
|
||||
$ pip install gevent
|
||||
$ pip install gevent
|
||||
|
||||
@@ -47,5 +47,3 @@ Requests License
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
|
||||
|
||||
|
||||
+22
-55
@@ -15,7 +15,7 @@ import config
|
||||
from .models import Request, Response, AuthObject
|
||||
from .status_codes import codes
|
||||
from .hooks import dispatch_hook
|
||||
from .utils import cookiejar_from_dict
|
||||
from .utils import cookiejar_from_dict, header_expand
|
||||
|
||||
|
||||
__all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete')
|
||||
@@ -24,8 +24,8 @@ def request(method, url,
|
||||
params=None, data=None, headers=None, cookies=None, files=None, auth=None,
|
||||
timeout=None, allow_redirects=False, proxies=None, hooks=None, return_response=True):
|
||||
|
||||
"""Constructs and sends a :class:`Request <models.Request>`.
|
||||
Returns :class:`Response <models.Response>` object.
|
||||
"""Constructs and sends a :class:`Request <Request>`.
|
||||
Returns :class:`Response <Response>` object.
|
||||
|
||||
:param method: method for the new :class:`Request` object.
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
@@ -38,13 +38,21 @@ def request(method, url,
|
||||
:param timeout: (optional) Float describing the timeout of the request.
|
||||
:param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed.
|
||||
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
|
||||
:param return_response: (optional) If False, an un-sent Request object will returned.
|
||||
"""
|
||||
|
||||
method = str(method).upper()
|
||||
|
||||
if cookies is None:
|
||||
cookies = {}
|
||||
|
||||
cookies = cookiejar_from_dict(cookies)
|
||||
|
||||
# Expand header values
|
||||
if headers:
|
||||
for k, v in headers.items() or {}:
|
||||
headers[k] = header_expand(v)
|
||||
|
||||
args = dict(
|
||||
method = method,
|
||||
url = url,
|
||||
@@ -89,31 +97,19 @@ def get(url, **kwargs):
|
||||
"""Sends a GET request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
|
||||
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
|
||||
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
|
||||
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
|
||||
:param timeout: (optional) Float describing the timeout of the request.
|
||||
:param allow_redirects: (optional) Boolean. Set to False to disable redirect following.
|
||||
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
|
||||
kwargs.setdefault('allow_redirects', True)
|
||||
return request('GET', url, **kwargs)
|
||||
|
||||
|
||||
def head(url, **kwargs):
|
||||
|
||||
"""Sends a HEAD request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
|
||||
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
|
||||
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
|
||||
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
|
||||
:param timeout: (optional) Float describing the timeout of the request.
|
||||
:param allow_redirects: (optional) Boolean. Set to False to disable redirect following.
|
||||
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
kwargs.setdefault('allow_redirects', True)
|
||||
@@ -121,22 +117,14 @@ def head(url, **kwargs):
|
||||
|
||||
|
||||
def post(url, data='', **kwargs):
|
||||
|
||||
"""Sends a POST request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
|
||||
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
|
||||
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
|
||||
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
|
||||
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
|
||||
:param timeout: (optional) Float describing the timeout of the request.
|
||||
:param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
|
||||
:param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
|
||||
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
return request('POST', url, data=data, **kwargs)
|
||||
return request('post', url, data=data, **kwargs)
|
||||
|
||||
|
||||
def put(url, data='', **kwargs):
|
||||
@@ -144,17 +132,10 @@ def put(url, data='', **kwargs):
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
|
||||
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
|
||||
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
|
||||
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
|
||||
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
|
||||
:param timeout: (optional) Float describing the timeout of the request.
|
||||
:param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
|
||||
:param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
|
||||
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
return request('PUT', url, data=data, **kwargs)
|
||||
return request('put', url, data=data, **kwargs)
|
||||
|
||||
|
||||
def patch(url, data='', **kwargs):
|
||||
@@ -162,31 +143,17 @@ def patch(url, data='', **kwargs):
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`.
|
||||
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
|
||||
:param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload.
|
||||
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
|
||||
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
|
||||
:param timeout: (optional) Float describing the timeout of the request.
|
||||
:param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
|
||||
:param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
|
||||
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
return request('PATCH', url, **kwargs)
|
||||
return request('patch', url, **kwargs)
|
||||
|
||||
|
||||
def delete(url, **kwargs):
|
||||
|
||||
"""Sends a DELETE request. Returns :class:`Response` object.
|
||||
|
||||
:param url: URL for the new :class:`Request` object.
|
||||
:param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`.
|
||||
:param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
|
||||
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
|
||||
:param auth: (optional) AuthObject to enable Basic HTTP Auth.
|
||||
:param timeout: (optional) Float describing the timeout of the request.
|
||||
:param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed.
|
||||
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
|
||||
:param **kwargs: Optional arguments that ``request`` takes.
|
||||
"""
|
||||
|
||||
return request('DELETE', url, **kwargs)
|
||||
return request('delete', url, **kwargs)
|
||||
|
||||
@@ -62,6 +62,7 @@ settings.proxies = None
|
||||
settings.verbose = None
|
||||
settings.timeout = None
|
||||
settings.max_redirects = 30
|
||||
settings.decode_unicode = True
|
||||
|
||||
#: Use socket.setdefaulttimeout() as fallback?
|
||||
settings.timeout_fallback = True
|
||||
|
||||
+4
-2
@@ -19,9 +19,11 @@ __license__ = 'ISC'
|
||||
__copyright__ = 'Copyright 2011 Kenneth Reitz'
|
||||
|
||||
|
||||
from models import HTTPError
|
||||
from models import HTTPError, Request, Response
|
||||
from api import *
|
||||
from exceptions import *
|
||||
from sessions import session
|
||||
from status_codes import codes
|
||||
from config import settings
|
||||
from config import settings
|
||||
|
||||
import utils
|
||||
|
||||
+102
-47
@@ -9,8 +9,10 @@ requests.models
|
||||
import urllib
|
||||
import urllib2
|
||||
import socket
|
||||
import codecs
|
||||
import zlib
|
||||
|
||||
|
||||
from urllib2 import HTTPError
|
||||
from urlparse import urlparse, urlunparse, urljoin
|
||||
from datetime import datetime
|
||||
@@ -20,9 +22,9 @@ from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicA
|
||||
from .structures import CaseInsensitiveDict
|
||||
from .packages.poster.encode import multipart_encode
|
||||
from .packages.poster.streaminghttp import register_openers, get_handlers
|
||||
from .utils import dict_from_cookiejar
|
||||
from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects
|
||||
from .utils import dict_from_cookiejar, get_unicode_from_response, stream_decode_response_unicode, decode_gzip, stream_decode_gzip
|
||||
from .status_codes import codes
|
||||
from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects
|
||||
|
||||
|
||||
REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
|
||||
@@ -30,7 +32,7 @@ REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)
|
||||
|
||||
|
||||
class Request(object):
|
||||
"""The :class:`Request <models.Request>` object. It carries out all functionality of
|
||||
"""The :class:`Request <Request>` object. It carries out all functionality of
|
||||
Requests. Recommended interface is with the Requests functions.
|
||||
"""
|
||||
|
||||
@@ -46,24 +48,24 @@ class Request(object):
|
||||
#: Request URL.
|
||||
self.url = url
|
||||
|
||||
#: Dictonary of HTTP Headers to attach to the :class:`Request <models.Request>`.
|
||||
#: Dictonary of HTTP Headers to attach to the :class:`Request <Request>`.
|
||||
self.headers = headers
|
||||
|
||||
#: Dictionary of files to multipart upload (``{filename: content}``).
|
||||
self.files = files
|
||||
|
||||
#: HTTP Method to use. Available: GET, HEAD, PUT, POST, DELETE.
|
||||
#: HTTP Method to use.
|
||||
self.method = method
|
||||
|
||||
#: Dictionary or byte of request body data to attach to the
|
||||
#: :class:`Request <models.Request>`.
|
||||
#: :class:`Request <Request>`.
|
||||
self.data = None
|
||||
|
||||
#: Dictionary or byte of querystring data to attach to the
|
||||
#: :class:`Request <models.Request>`.
|
||||
#: :class:`Request <Request>`.
|
||||
self.params = None
|
||||
|
||||
#: True if :class:`Request <models.Request>` is part of a redirect chain (disables history
|
||||
#: True if :class:`Request <Request>` is part of a redirect chain (disables history
|
||||
#: and HTTPError storage).
|
||||
self.redirect = redirect
|
||||
|
||||
@@ -76,7 +78,7 @@ class Request(object):
|
||||
self.data, self._enc_data = self._encode_params(data)
|
||||
self.params, self._enc_params = self._encode_params(params)
|
||||
|
||||
#: :class:`Response <models.Response>` instance, containing
|
||||
#: :class:`Response <Response>` instance, containing
|
||||
#: content and metadata of HTTP Response, once :attr:`sent <send>`.
|
||||
self.response = Response()
|
||||
|
||||
@@ -85,10 +87,10 @@ class Request(object):
|
||||
if not auth:
|
||||
auth = auth_manager.get_auth(self.url)
|
||||
|
||||
#: :class:`AuthObject` to attach to :class:`Request <models.Request>`.
|
||||
#: :class:`AuthObject` to attach to :class:`Request <Request>`.
|
||||
self.auth = auth
|
||||
|
||||
#: CookieJar to attach to :class:`Request <models.Request>`.
|
||||
#: CookieJar to attach to :class:`Request <Request>`.
|
||||
self.cookiejar = cookiejar
|
||||
|
||||
#: True if Request has been sent.
|
||||
@@ -134,9 +136,16 @@ class Request(object):
|
||||
_handlers.append(urllib2.HTTPCookieProcessor(self.cookiejar))
|
||||
|
||||
if self.auth:
|
||||
if not isinstance(self.auth.handler, (urllib2.AbstractBasicAuthHandler, urllib2.AbstractDigestAuthHandler)):
|
||||
if not isinstance(self.auth.handler,
|
||||
(urllib2.AbstractBasicAuthHandler,
|
||||
urllib2.AbstractDigestAuthHandler)):
|
||||
|
||||
# TODO: REMOVE THIS COMPLETELY
|
||||
auth_manager.add_password(self.auth.realm, self.url, self.auth.username, self.auth.password)
|
||||
auth_manager.add_password(
|
||||
self.auth.realm, self.url,
|
||||
self.auth.username,
|
||||
self.auth.password)
|
||||
|
||||
self.auth.handler = self.auth.handler(auth_manager)
|
||||
auth_manager.add_auth(self.url, self.auth)
|
||||
|
||||
@@ -168,7 +177,10 @@ class Request(object):
|
||||
|
||||
|
||||
def _build_response(self, resp, is_error=False):
|
||||
"""Build internal :class:`Response <models.Response>` object from given response."""
|
||||
"""Build internal :class:`Response <Response>` object
|
||||
from given response.
|
||||
"""
|
||||
|
||||
|
||||
def build(resp):
|
||||
|
||||
@@ -177,9 +189,7 @@ class Request(object):
|
||||
|
||||
try:
|
||||
response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None))
|
||||
response.read = resp.read
|
||||
response._resp = resp
|
||||
response._close = resp.close
|
||||
response.raw = resp
|
||||
|
||||
if self.cookiejar:
|
||||
response.cookies = dict_from_cookiejar(self.cookiejar)
|
||||
@@ -207,7 +217,7 @@ class Request(object):
|
||||
((r.status_code is codes.see_other) or (self.allow_redirects))
|
||||
):
|
||||
|
||||
r.close()
|
||||
r.raw.close()
|
||||
|
||||
if not len(history) < settings.max_redirects:
|
||||
raise TooManyRedirects()
|
||||
@@ -256,8 +266,8 @@ class Request(object):
|
||||
|
||||
Otherwise, assumes the data is already encoded appropriately, and
|
||||
returns it twice.
|
||||
|
||||
"""
|
||||
|
||||
if hasattr(data, 'items'):
|
||||
result = []
|
||||
for k, vs in data.items():
|
||||
@@ -301,7 +311,6 @@ class Request(object):
|
||||
"""
|
||||
|
||||
self._checks()
|
||||
success = False
|
||||
|
||||
# Logging
|
||||
if settings.verbose:
|
||||
@@ -367,7 +376,6 @@ class Request(object):
|
||||
|
||||
self._build_response(why, is_error=True)
|
||||
|
||||
|
||||
else:
|
||||
self._build_response(resp)
|
||||
self.response.ok = True
|
||||
@@ -378,37 +386,46 @@ class Request(object):
|
||||
return self.sent
|
||||
|
||||
|
||||
|
||||
class Response(object):
|
||||
"""The core :class:`Response <models.Response>` object. All
|
||||
:class:`Request <models.Request>` objects contain a
|
||||
:class:`response <models.Response>` attribute, which is an instance
|
||||
"""The core :class:`Response <Response>` object. All
|
||||
:class:`Request <Request>` objects contain a
|
||||
:class:`response <Response>` attribute, which is an instance
|
||||
of this class.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
#: Raw content of the response, in bytes.
|
||||
#: If ``content-encoding`` of response was set to ``gzip``, the
|
||||
#: response data will be automatically deflated.
|
||||
|
||||
self._content = None
|
||||
self._content_consumed = False
|
||||
|
||||
#: Integer Code of responded HTTP Status.
|
||||
self.status_code = None
|
||||
|
||||
#: Case-insensitive Dictionary of Response Headers.
|
||||
#: For example, ``headers['content-encoding']`` will return the
|
||||
#: value of a ``'Content-Encoding'`` response header.
|
||||
self.headers = CaseInsensitiveDict()
|
||||
|
||||
#: File-like object representation of response (for advanced usage).
|
||||
self.raw = None
|
||||
|
||||
#: Final URL location of Response.
|
||||
self.url = None
|
||||
|
||||
#: True if no :attr:`error` occured.
|
||||
self.ok = False
|
||||
|
||||
#: Resulting :class:`HTTPError` of request, if one occured.
|
||||
self.error = None
|
||||
#: A list of :class:`Response <models.Response>` objects from
|
||||
|
||||
#: A list of :class:`Response <Response>` objects from
|
||||
#: the history of the Request. Any redirect responses will end
|
||||
#: up here.
|
||||
self.history = []
|
||||
#: The Request that created the Response.
|
||||
|
||||
#: The :class:`Request <Request>` that created the Response.
|
||||
self.request = None
|
||||
|
||||
#: A dictionary of Cookies the server sent back.
|
||||
self.cookies = None
|
||||
|
||||
@@ -419,23 +436,65 @@ class Response(object):
|
||||
|
||||
def __nonzero__(self):
|
||||
"""Returns true if :attr:`status_code` is 'OK'."""
|
||||
|
||||
return not self.error
|
||||
|
||||
def iter_content(self, chunk_size=10 * 1024, decode_unicode=None):
|
||||
"""Iterates over the response data. This avoids reading the content
|
||||
at once into memory for large responses. The chunk size is the number
|
||||
of bytes it should read into memory. This is not necessarily the
|
||||
length of each item returned as decoding can take place.
|
||||
"""
|
||||
if self._content_consumed:
|
||||
raise RuntimeError('The content for this response was '
|
||||
'already consumed')
|
||||
|
||||
def __getattr__(self, name):
|
||||
"""Read and returns the full stream when accessing to :attr: `content`"""
|
||||
if name == 'content':
|
||||
if self._content is not None:
|
||||
return self._content
|
||||
self._content = self.read()
|
||||
if self.headers.get('content-encoding', '') == 'gzip':
|
||||
try:
|
||||
self._content = zlib.decompress(self._content, 16+zlib.MAX_WBITS)
|
||||
except zlib.error:
|
||||
pass
|
||||
def generate():
|
||||
while 1:
|
||||
chunk = self.raw.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
self._content_consumed = True
|
||||
gen = generate()
|
||||
if 'gzip' in self.headers.get('content-encoding', ''):
|
||||
gen = stream_decode_gzip(gen)
|
||||
if decode_unicode is None:
|
||||
decode_unicode = settings.decode_unicode
|
||||
if decode_unicode:
|
||||
gen = stream_decode_response_unicode(gen, self)
|
||||
return gen
|
||||
|
||||
@property
|
||||
def content(self):
|
||||
"""Content of the response, in bytes or unicode
|
||||
(if available).
|
||||
"""
|
||||
|
||||
if self._content is not None:
|
||||
return self._content
|
||||
else:
|
||||
raise AttributeError
|
||||
|
||||
if self._content_consumed:
|
||||
raise RuntimeError('The content for this response was '
|
||||
'already consumed')
|
||||
|
||||
# Read the contents.
|
||||
self._content = self.raw.read()
|
||||
|
||||
# Decode GZip'd content.
|
||||
if 'gzip' in self.headers.get('content-encoding', ''):
|
||||
try:
|
||||
self._content = decode_gzip(self._content)
|
||||
except zlib.error:
|
||||
pass
|
||||
|
||||
# Decode unicode content.
|
||||
if settings.decode_unicode:
|
||||
self._content = get_unicode_from_response(self)
|
||||
|
||||
self._content_consumed = True
|
||||
return self._content
|
||||
|
||||
|
||||
def raise_for_status(self):
|
||||
"""Raises stored :class:`HTTPError` or :class:`URLError`, if one occured."""
|
||||
@@ -443,10 +502,6 @@ class Response(object):
|
||||
raise self.error
|
||||
|
||||
|
||||
def close(self):
|
||||
if self._resp.fp is not None and hasattr(self._resp.fp, '_sock'):
|
||||
self._resp.fp._sock.recv = None
|
||||
self._close()
|
||||
|
||||
class AuthManager(object):
|
||||
"""Requests Authentication Manager."""
|
||||
|
||||
+187
-5
@@ -9,15 +9,69 @@ that are also useful for external consumption.
|
||||
|
||||
"""
|
||||
|
||||
import cgi
|
||||
import codecs
|
||||
import cookielib
|
||||
import re
|
||||
import zlib
|
||||
|
||||
|
||||
def dict_from_cookiejar(cookiejar):
|
||||
"""Returns a key/value dictionary from a CookieJar."""
|
||||
def header_expand(headers):
|
||||
"""Returns an HTTP Header value string from a dictionary.
|
||||
|
||||
Example expansion::
|
||||
|
||||
{'text/x-dvi': {'q': '.8', 'mxb': '100000', 'mxt': '5.0'}, 'text/x-c': {}}
|
||||
# Accept: text/x-dvi; q=.8; mxb=100000; mxt=5.0, text/x-c
|
||||
|
||||
(('text/x-dvi', {'q': '.8', 'mxb': '100000', 'mxt': '5.0'}), ('text/x-c', {}))
|
||||
# Accept: text/x-dvi; q=.8; mxb=100000; mxt=5.0, text/x-c
|
||||
"""
|
||||
|
||||
collector = []
|
||||
|
||||
if isinstance(headers, dict):
|
||||
headers = headers.items()
|
||||
|
||||
elif isinstance(headers, basestring):
|
||||
return headers
|
||||
|
||||
for i, (value, params) in enumerate(headers):
|
||||
|
||||
_params = []
|
||||
|
||||
for (p_k, p_v) in params.items():
|
||||
|
||||
_params.append('%s=%s' % (p_k, p_v))
|
||||
|
||||
collector.append(value)
|
||||
collector.append('; ')
|
||||
|
||||
if len(params):
|
||||
|
||||
collector.append('; '.join(_params))
|
||||
|
||||
if not len(headers) == i+1:
|
||||
collector.append(', ')
|
||||
|
||||
|
||||
# Remove trailing seperators.
|
||||
if collector[-1] in (', ', '; '):
|
||||
del collector[-1]
|
||||
|
||||
return ''.join(collector)
|
||||
|
||||
|
||||
|
||||
def dict_from_cookiejar(cj):
|
||||
"""Returns a key/value dictionary from a CookieJar.
|
||||
|
||||
:param cj: CookieJar object to extract cookies from.
|
||||
"""
|
||||
|
||||
cookie_dict = {}
|
||||
|
||||
for _, cookies in cookiejar._cookies.items():
|
||||
for _, cookies in cj._cookies.items():
|
||||
for _, cookies in cookies.items():
|
||||
for cookie in cookies.values():
|
||||
# print cookie
|
||||
@@ -27,7 +81,10 @@ def dict_from_cookiejar(cookiejar):
|
||||
|
||||
|
||||
def cookiejar_from_dict(cookie_dict):
|
||||
"""Returns a CookieJar from a key/value dictionary."""
|
||||
"""Returns a CookieJar from a key/value dictionary.
|
||||
|
||||
:param cookie_dict: Dict of key/values to insert into CookieJar.
|
||||
"""
|
||||
|
||||
# return cookiejar if one was passed in
|
||||
if isinstance(cookie_dict, cookielib.CookieJar):
|
||||
@@ -42,7 +99,11 @@ def cookiejar_from_dict(cookie_dict):
|
||||
|
||||
|
||||
def add_dict_to_cookiejar(cj, cookie_dict):
|
||||
"""Returns a CookieJar from a key/value dictionary."""
|
||||
"""Returns a CookieJar from a key/value dictionary.
|
||||
|
||||
:param cj: CookieJar to insert cookies into.
|
||||
:param cookie_dict: Dict of key/values to insert into CookieJar.
|
||||
"""
|
||||
|
||||
for k, v in cookie_dict.items():
|
||||
|
||||
@@ -70,3 +131,124 @@ def add_dict_to_cookiejar(cj, cookie_dict):
|
||||
cj.set_cookie(cookie)
|
||||
|
||||
return cj
|
||||
|
||||
|
||||
def get_encodings_from_content(content):
|
||||
"""Returns encodings from given content string.
|
||||
|
||||
:param content: bytestring to extract encodings from.
|
||||
"""
|
||||
|
||||
charset_re = re.compile(r'<meta.*?charset=["\']*(.+?)["\'>]', flags=re.I)
|
||||
|
||||
return charset_re.findall(content)
|
||||
|
||||
|
||||
def get_encoding_from_headers(headers):
|
||||
"""Returns encodings from given HTTP Header Dict.
|
||||
|
||||
:param headers: dictionary to extract encoding from.
|
||||
"""
|
||||
|
||||
content_type = headers.get('content-type')
|
||||
|
||||
if not content_type:
|
||||
return None
|
||||
|
||||
content_type, params = cgi.parse_header(content_type)
|
||||
|
||||
if 'charset' in params:
|
||||
return params['charset'].strip("'\"")
|
||||
|
||||
|
||||
def unicode_from_html(content):
|
||||
"""Attempts to decode an HTML string into unicode.
|
||||
If unsuccessful, the original content is returned.
|
||||
"""
|
||||
|
||||
encodings = get_encodings_from_content(content)
|
||||
|
||||
for encoding in encodings:
|
||||
|
||||
try:
|
||||
return unicode(content, encoding)
|
||||
except (UnicodeError, TypeError):
|
||||
pass
|
||||
|
||||
return content
|
||||
|
||||
|
||||
def stream_decode_response_unicode(iterator, r):
|
||||
"""Stream decodes a iterator."""
|
||||
encoding = get_encoding_from_headers(r.headers)
|
||||
if encoding is None:
|
||||
for item in iterator:
|
||||
yield item
|
||||
return
|
||||
|
||||
decoder = codecs.getincrementaldecoder(encoding)(errors='replace')
|
||||
for chunk in iterator:
|
||||
rv = decoder.decode(chunk)
|
||||
if rv:
|
||||
yield rv
|
||||
rv = decoder.decode('', final=True)
|
||||
if rv:
|
||||
yield rv
|
||||
|
||||
|
||||
def get_unicode_from_response(r):
|
||||
"""Returns the requested content back in unicode.
|
||||
|
||||
:param r: Reponse object to get unicode content from.
|
||||
|
||||
Tried:
|
||||
|
||||
1. charset from content-type
|
||||
|
||||
2. every encodings from ``<meta ... charset=XXX>``
|
||||
|
||||
3. fall back and replace all unicode characters
|
||||
|
||||
"""
|
||||
|
||||
tried_encodings = []
|
||||
|
||||
# Try charset from content-type
|
||||
encoding = get_encoding_from_headers(r.headers)
|
||||
|
||||
if encoding:
|
||||
try:
|
||||
return unicode(r.content, encoding)
|
||||
except UnicodeError:
|
||||
tried_encodings.append(encoding)
|
||||
|
||||
# Fall back:
|
||||
try:
|
||||
return unicode(r.content, encoding, errors='replace')
|
||||
except TypeError:
|
||||
return r.content
|
||||
|
||||
|
||||
def decode_gzip(content):
|
||||
"""Return gzip-decoded string.
|
||||
|
||||
:param content: bytestring to gzip-decode.
|
||||
"""
|
||||
|
||||
return zlib.decompress(content, 16 + zlib.MAX_WBITS)
|
||||
|
||||
|
||||
def stream_decode_gzip(iterator):
|
||||
"""Stream decodes a gzip-encoded iterator"""
|
||||
try:
|
||||
dec = zlib.decompressobj(16 + zlib.MAX_WBITS)
|
||||
for chunk in iterator:
|
||||
rv = dec.decompress(chunk)
|
||||
if rv:
|
||||
yield rv
|
||||
buf = dec.decompress('')
|
||||
rv = buf + dec.flush()
|
||||
if rv:
|
||||
yield rv
|
||||
except zlib.error:
|
||||
pass
|
||||
|
||||
+7
-1
@@ -62,6 +62,13 @@ class RequestsTestSuite(unittest.TestCase):
|
||||
r = requests.get(httpbin('/'))
|
||||
self.assertEqual(r.status_code, 200)
|
||||
|
||||
def test_HTTP_302_ALLOW_REDIRECT_GET(self):
|
||||
r = requests.get(httpbin('redirect', '1'))
|
||||
self.assertEqual(r.status_code, 200)
|
||||
|
||||
def test_HTTP_302_GET(self):
|
||||
r = requests.get(httpbin('redirect', '1'), allow_redirects=False)
|
||||
self.assertEqual(r.status_code, 302)
|
||||
|
||||
def test_HTTPS_200_OK_GET(self):
|
||||
r = requests.get(httpsbin('/'))
|
||||
@@ -490,6 +497,5 @@ class RequestsTestSuite(unittest.TestCase):
|
||||
self.assertEqual(r2.status_code, 200)
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user