mirror of
https://github.com/not-kennethreitz/requests-async.git
synced 2026-06-05 15:00:20 +00:00
Tom Christie
287 lines
10 KiB
Python
287 lines
10 KiB
Python
import datetime
|
|
import requests
|
|
from . import adapters
|
|
from requests.exceptions import TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError
|
|
from requests.cookies import extract_cookies_to_jar, merge_cookies
|
|
from requests.status_codes import codes
|
|
from requests.utils import requote_uri
|
|
from urllib.parse import urlparse
|
|
|
|
|
|
def to_native_string(string, encoding='ascii'):
|
|
"""Given a string object, regardless of type, returns a representation of
|
|
that string in the native string type, encoding and decoding where
|
|
necessary. This assumes ASCII unless told otherwise.
|
|
"""
|
|
if isinstance(string, str):
|
|
return string
|
|
return string.decode(encoding)
|
|
|
|
|
|
class Session(requests.Session):
|
|
def __init__(self, *args, **kwargs) -> None:
|
|
super(Session, self).__init__(*args, **kwargs)
|
|
adapter = adapters.HTTPAdapter()
|
|
self.mount("http://", adapter)
|
|
self.mount("https://", adapter)
|
|
|
|
async def request(
|
|
self,
|
|
method,
|
|
url,
|
|
params=None,
|
|
data=None,
|
|
headers=None,
|
|
cookies=None,
|
|
files=None,
|
|
auth=None,
|
|
timeout=None,
|
|
allow_redirects=True,
|
|
proxies=None,
|
|
hooks=None,
|
|
stream=None,
|
|
verify=None,
|
|
cert=None,
|
|
json=None,
|
|
):
|
|
# Create the Request.
|
|
req = requests.models.Request(
|
|
method=method.upper(),
|
|
url=url,
|
|
headers=headers,
|
|
files=files,
|
|
data=data or {},
|
|
json=json,
|
|
params=params or {},
|
|
auth=auth,
|
|
cookies=cookies,
|
|
hooks=hooks,
|
|
)
|
|
prep = self.prepare_request(req)
|
|
|
|
proxies = proxies or {}
|
|
|
|
settings = self.merge_environment_settings(
|
|
prep.url, proxies, stream, verify, cert
|
|
)
|
|
|
|
# Send the request.
|
|
send_kwargs = {"timeout": timeout, "allow_redirects": allow_redirects}
|
|
send_kwargs.update(settings)
|
|
resp = await self.send(prep, **send_kwargs)
|
|
|
|
return resp
|
|
|
|
async def get(self, url, **kwargs):
|
|
kwargs.setdefault("allow_redirects", True)
|
|
return await self.request("GET", url, **kwargs)
|
|
|
|
async def options(self, url, **kwargs):
|
|
kwargs.setdefault("allow_redirects", True)
|
|
return await self.request("OPTIONS", url, **kwargs)
|
|
|
|
async def head(self, url, **kwargs):
|
|
kwargs.setdefault("allow_redirects", False)
|
|
return await self.request("HEAD", url, **kwargs)
|
|
|
|
async def post(self, url, data=None, json=None, **kwargs):
|
|
return await self.request("POST", url, data=data, json=json, **kwargs)
|
|
|
|
async def put(self, url, data=None, **kwargs):
|
|
return await self.request("PUT", url, data=data, **kwargs)
|
|
|
|
async def patch(self, url, data=None, **kwargs):
|
|
return await self.request("PATCH", url, data=data, **kwargs)
|
|
|
|
async def delete(self, url, **kwargs):
|
|
return await self.request("DELETE", url, **kwargs)
|
|
|
|
async def send(self, request, **kwargs):
|
|
"""Send a given PreparedRequest.
|
|
|
|
:rtype: requests.Response
|
|
"""
|
|
# Set defaults that the hooks can utilize to ensure they always have
|
|
# the correct parameters to reproduce the previous request.
|
|
kwargs.setdefault("stream", self.stream)
|
|
kwargs.setdefault("verify", self.verify)
|
|
kwargs.setdefault("cert", self.cert)
|
|
kwargs.setdefault("proxies", self.proxies)
|
|
|
|
# It's possible that users might accidentally send a Request object.
|
|
# Guard against that specific failure case.
|
|
if isinstance(request, requests.models.Request):
|
|
raise ValueError("You can only send PreparedRequests.")
|
|
|
|
# Set up variables needed for resolve_redirects and dispatching of hooks
|
|
allow_redirects = kwargs.pop("allow_redirects", True)
|
|
stream = kwargs.get("stream")
|
|
hooks = request.hooks
|
|
|
|
# Get the appropriate adapter to use
|
|
adapter = self.get_adapter(url=request.url)
|
|
|
|
# Start time (approximately) of the request
|
|
start = requests.sessions.preferred_clock()
|
|
|
|
# Send the request
|
|
r = await adapter.send(request, **kwargs)
|
|
|
|
# Total elapsed time of the request (approximately)
|
|
elapsed = requests.sessions.preferred_clock() - start
|
|
r.elapsed = datetime.timedelta(seconds=elapsed)
|
|
|
|
# Response manipulation hooks
|
|
r = requests.hooks.dispatch_hook("response", hooks, r, **kwargs)
|
|
|
|
# Persist cookies
|
|
if r.history:
|
|
|
|
# If the hooks create history then we want those cookies too
|
|
for resp in r.history:
|
|
requests.cookies.extract_cookies_to_jar(
|
|
self.cookies, resp.request, resp.raw
|
|
)
|
|
|
|
requests.cookies.extract_cookies_to_jar(self.cookies, request, r.raw)
|
|
|
|
# Redirect resolving.
|
|
history = []
|
|
if allow_redirects:
|
|
async for resp in self.resolve_redirects(r, request, **kwargs):
|
|
history.append(resp)
|
|
|
|
# Shuffle things around if there's history.
|
|
if history:
|
|
# Insert the first (original) request at the start
|
|
history.insert(0, r)
|
|
# Get the last request made
|
|
r = history.pop()
|
|
r.history = history
|
|
|
|
# If redirects aren't being followed, store the response on the Request for Response.next().
|
|
# if not allow_redirects:
|
|
# try:
|
|
# r._next = next(
|
|
# self.resolve_redirects(r, request, yield_requests=True, **kwargs)
|
|
# )
|
|
# except StopIteration:
|
|
# pass
|
|
|
|
if not stream:
|
|
r.content
|
|
|
|
return r
|
|
|
|
async def resolve_redirects(self, resp, req, stream=False, timeout=None,
|
|
verify=True, cert=None, proxies=None, yield_requests=False, **adapter_kwargs):
|
|
"""Receives a Response. Returns a generator of Responses or Requests."""
|
|
hist = [] # keep track of history
|
|
|
|
url = self.get_redirect_target(resp)
|
|
previous_fragment = urlparse(req.url).fragment
|
|
while url:
|
|
prepared_request = req.copy()
|
|
|
|
# Update history and keep track of redirects.
|
|
# resp.history must ignore the original request in this loop
|
|
hist.append(resp)
|
|
resp.history = hist[1:]
|
|
|
|
try:
|
|
resp.content # Consume socket so it can be released
|
|
except (ChunkedEncodingError, ContentDecodingError, RuntimeError):
|
|
resp.raw.read(decode_content=False)
|
|
|
|
if len(resp.history) >= self.max_redirects:
|
|
raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects, response=resp)
|
|
|
|
# Release the connection back into the pool.
|
|
resp.close()
|
|
|
|
# Handle redirection without scheme (see: RFC 1808 Section 4)
|
|
if url.startswith('//'):
|
|
parsed_rurl = urlparse(resp.url)
|
|
url = '%s:%s' % (to_native_string(parsed_rurl.scheme), url)
|
|
|
|
# Normalize url case and attach previous fragment if needed (RFC 7231 7.1.2)
|
|
parsed = urlparse(url)
|
|
if parsed.fragment == '' and previous_fragment:
|
|
parsed = parsed._replace(fragment=previous_fragment)
|
|
elif parsed.fragment:
|
|
previous_fragment = parsed.fragment
|
|
url = parsed.geturl()
|
|
|
|
# Facilitate relative 'location' headers, as allowed by RFC 7231.
|
|
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
|
|
# Compliant with RFC3986, we percent encode the url.
|
|
if not parsed.netloc:
|
|
url = urljoin(resp.url, requote_uri(url))
|
|
else:
|
|
url = requote_uri(url)
|
|
|
|
prepared_request.url = to_native_string(url)
|
|
|
|
self.rebuild_method(prepared_request, resp)
|
|
|
|
# https://github.com/requests/requests/issues/1084
|
|
if resp.status_code not in (codes.temporary_redirect, codes.permanent_redirect):
|
|
# https://github.com/requests/requests/issues/3490
|
|
purged_headers = ('Content-Length', 'Content-Type', 'Transfer-Encoding')
|
|
for header in purged_headers:
|
|
prepared_request.headers.pop(header, None)
|
|
prepared_request.body = None
|
|
|
|
headers = prepared_request.headers
|
|
try:
|
|
del headers['Cookie']
|
|
except KeyError:
|
|
pass
|
|
|
|
# Extract any cookies sent on the response to the cookiejar
|
|
# in the new request. Because we've mutated our copied prepared
|
|
# request, use the old one that we haven't yet touched.
|
|
extract_cookies_to_jar(prepared_request._cookies, req, resp.raw)
|
|
merge_cookies(prepared_request._cookies, self.cookies)
|
|
prepared_request.prepare_cookies(prepared_request._cookies)
|
|
|
|
# Rebuild auth and proxy information.
|
|
proxies = self.rebuild_proxies(prepared_request, proxies)
|
|
self.rebuild_auth(prepared_request, resp)
|
|
|
|
# A failed tell() sets `_body_position` to `object()`. This non-None
|
|
# value ensures `rewindable` will be True, allowing us to raise an
|
|
# UnrewindableBodyError, instead of hanging the connection.
|
|
rewindable = (
|
|
prepared_request._body_position is not None and
|
|
('Content-Length' in headers or 'Transfer-Encoding' in headers)
|
|
)
|
|
|
|
# Attempt to rewind consumed file-like object.
|
|
if rewindable:
|
|
rewind_body(prepared_request)
|
|
|
|
# Override the original request.
|
|
req = prepared_request
|
|
|
|
if yield_requests:
|
|
yield req
|
|
else:
|
|
|
|
resp = await self.send(
|
|
req,
|
|
stream=stream,
|
|
timeout=timeout,
|
|
verify=verify,
|
|
cert=cert,
|
|
proxies=proxies,
|
|
allow_redirects=False,
|
|
**adapter_kwargs
|
|
)
|
|
|
|
extract_cookies_to_jar(self.cookies, prepared_request, resp.raw)
|
|
|
|
# extract redirect url, if any, for the next loop
|
|
url = self.get_redirect_target(resp)
|
|
yield resp
|