From 66be821d130862cec41436e272a5a9df7c68716d Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 21 Mar 2019 11:26:42 +0000 Subject: [PATCH] Initial commit --- .gitignore | 8 ++ .travis.yml | 20 +++++ LICENSE.md | 27 +++++++ README.md | 33 ++++++++ requests_async/__init__.py | 16 ++++ requests_async/adapters.py | 81 +++++++++++++++++++ requests_async/api.py | 37 +++++++++ requests_async/sessions.py | 159 +++++++++++++++++++++++++++++++++++++ requirements.txt | 12 +++ scripts/clean | 14 ++++ scripts/lint | 12 +++ scripts/publish | 28 +++++++ scripts/test | 12 +++ setup.py | 63 +++++++++++++++ tests/conftest.py | 52 ++++++++++++ tests/test_api.py | 75 +++++++++++++++++ tests/test_sessions.py | 37 +++++++++ 17 files changed, 686 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 LICENSE.md create mode 100644 README.md create mode 100644 requests_async/__init__.py create mode 100644 requests_async/adapters.py create mode 100644 requests_async/api.py create mode 100644 requests_async/sessions.py create mode 100644 requirements.txt create mode 100755 scripts/clean create mode 100755 scripts/lint create mode 100755 scripts/publish create mode 100755 scripts/test create mode 100644 setup.py create mode 100644 tests/conftest.py create mode 100644 tests/test_api.py create mode 100644 tests/test_sessions.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..938e2d1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +*.pyc +.coverage +.pytest_cache/ +.mypy_cache/ +__pycache__/ +htmlcov/ +*.egg-info/ +venv/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..ff26a2b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,20 @@ +sudo: required +dist: xenial +language: python + +cache: pip + +python: + - "3.6" + - "3.7" + - "3.8-dev" + +install: + - pip install -U -r requirements.txt + +script: + - scripts/test + +after_script: + - pip install codecov + - codecov diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..8963b9f --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,27 @@ +Copyright © 2019, [Encode OSS Ltd](https://www.encode.io/). +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b28f625 --- /dev/null +++ b/README.md @@ -0,0 +1,33 @@ +# requests-async + +Brings support for `async`/`await` syntax to Python's fabulous `requests` library. + +**This is just a first-pass right now.** + +Next set of things to deal with: + +* https support, and certificate checking. +* streaming support for uploads and downloads. +* connection pooling. +* async redirections. +* async cookie persistence, for on-disk cookie stores. +* make sure authentication works okay (does it use adapters, is the API broken there now?) + +Installation: + +```shell +$ pip install requests-async +``` + +Usage: + +Just use the standard requests API, but use `await` for making requests. + +```python +import requests_async as requests + + +response = await requests.get('http://example.org') +print(response.status_code) +print(response.text) +``` diff --git a/requests_async/__init__.py b/requests_async/__init__.py new file mode 100644 index 0000000..97fe470 --- /dev/null +++ b/requests_async/__init__.py @@ -0,0 +1,16 @@ +from .adapters import HTTPAdapter +from .sessions import Session +from .api import request, get, head, post, patch, put, delete, options + +__version__ = "0.0.1" +__all__ = [ + "request", + "get", + "head", + "post", + "patch", + "put", + "delete", + "options", + "Session", +] diff --git a/requests_async/adapters.py b/requests_async/adapters.py new file mode 100644 index 0000000..28b4944 --- /dev/null +++ b/requests_async/adapters.py @@ -0,0 +1,81 @@ +import asyncio +import io +import typing +from urllib.parse import urlparse + +import h11 +import requests +import urllib3 + + +class HTTPAdapter(requests.adapters.HTTPAdapter): + async def send( + self, request: requests.PreparedRequest, *args: typing.Any, **kwargs: typing.Any + ) -> requests.Response: + urlparts = urlparse(request.url) + + hostname = urlparts.hostname + port = urlparts.port + if port is None: + port = {"http": 80, "https": 443}[urlparts.scheme] + target = urlparts.path + if urlparts.query: + target += "?" + urlparts.query + headers = [("host", urlparts.netloc)] + list(request.headers.items()) + + reader, writer = await asyncio.open_connection(hostname, port) + + conn = h11.Connection(our_role=h11.CLIENT) + + message = h11.Request(method=request.method, target=target, headers=headers) + data = conn.send(message) + writer.write(data) + + if request.body: + message = h11.Data(data=request.body.encode("utf-8")) + data = conn.send(message) + writer.write(data) + + message = h11.EndOfMessage() + data = conn.send(message) + writer.write(data) + + status_code = 0 + headers = [] + reason = b"" + buffer = io.BytesIO() + + while True: + event = conn.next_event() + event_type = type(event) + + if event_type is h11.NEED_DATA: + data = await reader.read(2048) + conn.receive_data(data) + + elif event_type is h11.Response: + status_code = event.status_code + headers = [ + (key.decode(), value.decode()) for key, value in event.headers + ] + reason = event.reason + + elif event_type is h11.Data: + buffer.write(event.data) + + elif event_type is h11.EndOfMessage: + buffer.seek(0) + break + + writer.close() + await writer.wait_closed() + + resp = urllib3.HTTPResponse( + body=buffer, + headers=headers, + status=status_code, + reason=reason, + preload_content=False, + ) + + return self.build_response(request, resp) diff --git a/requests_async/api.py b/requests_async/api.py new file mode 100644 index 0000000..0b0b04c --- /dev/null +++ b/requests_async/api.py @@ -0,0 +1,37 @@ +from . import sessions + + +async def request(method, url, **kwargs): + with sessions.Session() as session: + return await session.request(method=method, url=url, **kwargs) + + +async def get(url, params=None, **kwargs): + kwargs.setdefault("allow_redirects", True) + return await request("get", url, params=params, **kwargs) + + +async def options(url, **kwargs): + kwargs.setdefault("allow_redirects", True) + return await request("options", url, **kwargs) + + +async def head(url, **kwargs): + kwargs.setdefault("allow_redirects", False) + return await request("head", url, **kwargs) + + +async def post(url, data=None, json=None, **kwargs): + return await request("post", url, data=data, json=json, **kwargs) + + +async def put(url, data=None, **kwargs): + return await request("put", url, data=data, **kwargs) + + +async def patch(url, data=None, **kwargs): + return await request("patch", url, data=data, **kwargs) + + +async def delete(url, **kwargs): + return await request("delete", url, **kwargs) diff --git a/requests_async/sessions.py b/requests_async/sessions.py new file mode 100644 index 0000000..785833a --- /dev/null +++ b/requests_async/sessions.py @@ -0,0 +1,159 @@ +import datetime +import requests +from . import adapters + + +class Session(requests.Session): + def __init__(self, *args, **kwargs) -> None: + super(Session, self).__init__(*args, **kwargs) + adapter = adapters.HTTPAdapter() + self.mount("http://", adapter) + self.mount("https://", adapter) + + async def request( + self, + method, + url, + params=None, + data=None, + headers=None, + cookies=None, + files=None, + auth=None, + timeout=None, + allow_redirects=True, + proxies=None, + hooks=None, + stream=None, + verify=None, + cert=None, + json=None, + ): + # Create the Request. + req = requests.models.Request( + method=method.upper(), + url=url, + headers=headers, + files=files, + data=data or {}, + json=json, + params=params or {}, + auth=auth, + cookies=cookies, + hooks=hooks, + ) + prep = self.prepare_request(req) + + proxies = proxies or {} + + settings = self.merge_environment_settings( + prep.url, proxies, stream, verify, cert + ) + + # Send the request. + send_kwargs = {"timeout": timeout, "allow_redirects": allow_redirects} + send_kwargs.update(settings) + resp = await self.send(prep, **send_kwargs) + + return resp + + async def get(self, url, **kwargs): + kwargs.setdefault("allow_redirects", True) + return await self.request("GET", url, **kwargs) + + async def options(self, url, **kwargs): + kwargs.setdefault("allow_redirects", True) + return await self.request("OPTIONS", url, **kwargs) + + async def head(self, url, **kwargs): + kwargs.setdefault("allow_redirects", False) + return await self.request("HEAD", url, **kwargs) + + async def post(self, url, data=None, json=None, **kwargs): + return await self.request("POST", url, data=data, json=json, **kwargs) + + async def put(self, url, data=None, **kwargs): + return await self.request("PUT", url, data=data, **kwargs) + + async def patch(self, url, data=None, **kwargs): + return await self.request("PATCH", url, data=data, **kwargs) + + async def delete(self, url, **kwargs): + return await self.request("DELETE", url, **kwargs) + + async def send(self, request, **kwargs): + """Send a given PreparedRequest. + + :rtype: requests.Response + """ + # Set defaults that the hooks can utilize to ensure they always have + # the correct parameters to reproduce the previous request. + kwargs.setdefault("stream", self.stream) + kwargs.setdefault("verify", self.verify) + kwargs.setdefault("cert", self.cert) + kwargs.setdefault("proxies", self.proxies) + + # It's possible that users might accidentally send a Request object. + # Guard against that specific failure case. + if isinstance(request, requests.models.Request): + raise ValueError("You can only send PreparedRequests.") + + # Set up variables needed for resolve_redirects and dispatching of hooks + allow_redirects = kwargs.pop("allow_redirects", True) + stream = kwargs.get("stream") + hooks = request.hooks + + # Get the appropriate adapter to use + adapter = self.get_adapter(url=request.url) + + # Start time (approximately) of the request + start = requests.sessions.preferred_clock() + + # Send the request + r = await adapter.send(request, **kwargs) + + # Total elapsed time of the request (approximately) + elapsed = requests.sessions.preferred_clock() - start + r.elapsed = datetime.timedelta(seconds=elapsed) + + # Response manipulation hooks + r = requests.hooks.dispatch_hook("response", hooks, r, **kwargs) + + # Persist cookies + if r.history: + + # If the hooks create history then we want those cookies too + for resp in r.history: + requests.cookies.extract_cookies_to_jar( + self.cookies, resp.request, resp.raw + ) + + requests.cookies.extract_cookies_to_jar(self.cookies, request, r.raw) + + # Redirect resolving generator. + gen = self.resolve_redirects(r, request, **kwargs) + + # Resolve redirects if allowed. + history = [resp for resp in gen] if allow_redirects else [] + + # Shuffle things around if there's history. + if history: + # Insert the first (original) request at the start + history.insert(0, r) + # Get the last request made + r = history.pop() + r.history = history + + # If redirects aren't being followed, store the response on the Request for Response.next(). + if not allow_redirects: + try: + r._next = next( + self.resolve_redirects(r, request, yield_requests=True, **kwargs) + ) + except StopIteration: + pass + + if not stream: + r.content + + return r diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..59110bc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +h11 +requests + +# Testing +black +isort +pytest +pytest-asyncio +pytest-cov +python-multipart +starlette +uvicorn diff --git a/scripts/clean b/scripts/clean new file mode 100755 index 0000000..f6898f5 --- /dev/null +++ b/scripts/clean @@ -0,0 +1,14 @@ +#!/bin/sh -e + +if [ -d 'dist' ] ; then + rm -r dist +fi +if [ -d 'site' ] ; then + rm -r site +fi +if [ -d 'htmlcov' ] ; then + rm -r htmlcov +fi +if [ -d 'requests_async.egg-info' ] ; then + rm -r requests_async.egg-info +fi diff --git a/scripts/lint b/scripts/lint new file mode 100755 index 0000000..394ccbe --- /dev/null +++ b/scripts/lint @@ -0,0 +1,12 @@ +#!/bin/sh -e + +export PACKAGE="requests_async" +export PREFIX="" +if [ -d 'venv' ] ; then + export PREFIX="venv/bin/" +fi + +set -x + +${PREFIX}black ${PACKAGE} tests setup.py +${PREFIX}isort --multi-line=3 --trailing-comma --force-grid-wrap=0 --combine-as --line-width 88 --recursive --apply ${PACKAGE} tests setup.py diff --git a/scripts/publish b/scripts/publish new file mode 100755 index 0000000..94cc0b2 --- /dev/null +++ b/scripts/publish @@ -0,0 +1,28 @@ +#!/bin/sh -e + +export PACKAGE="requests_async" +export VERSION=`cat ${PACKAGE}/__init__.py | grep __version__ | sed "s/__version__ = //" | sed "s/'//g"` +export PREFIX="" +if [ -d 'venv' ] ; then + export PREFIX="venv/bin/" +fi + +scripts/clean + +if ! command -v "${PREFIX}twine" &>/dev/null ; then + echo "Unable to find the 'twine' command." + echo "Install from PyPI, using '${PREFIX}pip install twine'." + exit 1 +fi + +find ${PACKAGE} -type f -name "*.py[co]" -delete +find ${PACKAGE} -type d -name __pycache__ -delete + +${PREFIX}python setup.py sdist +${PREFIX}twine upload dist/* + +echo "You probably want to also tag the version now:" +echo "git tag -a ${VERSION} -m 'version ${VERSION}'" +echo "git push --tags" + +scripts/clean diff --git a/scripts/test b/scripts/test new file mode 100755 index 0000000..046f8ed --- /dev/null +++ b/scripts/test @@ -0,0 +1,12 @@ +#!/bin/sh -e + +export PACKAGE="requests_async" +export PREFIX="" +if [ -d 'venv' ] ; then + export PREFIX="venv/bin/" +fi + +set -x + +PYTHONPATH=. ${PREFIX}pytest --ignore venv --cov tests --cov ${PACKAGE} --cov-report= ${@} +${PREFIX}coverage report diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e20dbef --- /dev/null +++ b/setup.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import re + +from setuptools import setup + + +def get_version(package): + """ + Return package version as listed in `__version__` in `init.py`. + """ + with open(os.path.join(package, "__init__.py")) as f: + return re.search("__version__ = ['\"]([^'\"]+)['\"]", f.read()).group(1) + + +def get_long_description(): + """ + Return the README. + """ + with open("README.md", encoding="utf8") as f: + return f.read() + + +def get_packages(package): + """ + Return root package and all sub-packages. + """ + return [ + dirpath + for dirpath, dirnames, filenames in os.walk(package) + if os.path.exists(os.path.join(dirpath, "__init__.py")) + ] + + +setup( + name="requests-async", + python_requires=">=3.6", + version=get_version("requests_async"), + url="https://github.com/encode/requests-async", + license="BSD", + description="async-await support for `requests`.", + long_description=get_long_description(), + long_description_content_type="text/markdown", + author="Tom Christie", + author_email="tom@tomchristie.com", + packages=get_packages("requests_async"), + data_files=[("", ["LICENSE.md"])], + install_requires=["requests", "h11"], + classifiers=[ + "Development Status :: 3 - Alpha", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Topic :: Internet :: WWW/HTTP", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + ], +) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..0b206fd --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,52 @@ +import asyncio +import pytest + +from starlette.applications import Starlette +from starlette.responses import JSONResponse +from starlette.routing import Route + +from uvicorn.config import Config +from uvicorn.main import Server + + +async def echo_request(request): + body = await request.body() + return JSONResponse( + { + "method": request.method, + "url": str(request.url), + "body": body.decode("utf-8"), + } + ) + + +async def echo_form_data(request): + form = await request.form() + return JSONResponse( + { + "method": request.method, + "url": str(request.url), + "form": {key: value for key, value in form.items()}, + } + ) + + +routes = [ + Route("/", echo_request, methods=["GET", "DELETE", "OPTIONS", "POST", "PUT", "PATCH"]), + Route("/echo_form_data", echo_form_data, methods=["POST", "PUT", "PATCH"]), +] + +app = Starlette(routes=routes) + + +@pytest.fixture +async def server(): + config = Config(app=app, lifespan="off") + server = Server(config=config) + task = asyncio.create_task(server.serve()) + try: + while not server.started: + await asyncio.sleep(0.0001) + yield server + finally: + task.cancel() diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..8d1dcda --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,75 @@ +import asyncio +import requests_async +import pytest + + +@pytest.mark.asyncio +async def test_get(server): + url = "http://127.0.0.1:8000/" + response = await requests_async.get(url) + assert response.status_code == 200 + assert response.json() == {"method": "GET", "url": url, "body": ""} + + +@pytest.mark.asyncio +async def test_get_queryparams(server): + url = "http://127.0.0.1:8000/" + response = await requests_async.get(url, params={"a": "b"}) + assert response.status_code == 200 + assert response.json() == {"method": "GET", "url": url + "?a=b", "body": ""} + + +@pytest.mark.asyncio +async def test_head(server): + url = "http://127.0.0.1:8000/" + response = await requests_async.head(url) + assert response.status_code == 200 + assert response.text == "" + + +@pytest.mark.asyncio +async def test_options(server): + url = "http://127.0.0.1:8000/" + response = await requests_async.options(url) + assert response.status_code == 200 + assert response.json() == {"method": "OPTIONS", "url": url, "body": ""} + + +@pytest.mark.asyncio +async def test_delete(server): + url = "http://127.0.0.1:8000/" + response = await requests_async.delete(url) + assert response.status_code == 200 + assert response.json() == {"method": "DELETE", "url": url, "body": ""} + + +@pytest.mark.asyncio +async def test_post(server): + url = "http://127.0.0.1:8000/echo_form_data" + response = await requests_async.post(url) + assert response.status_code == 200 + assert response.json() == {"method": "POST", "url": url, "form": {}} + + +@pytest.mark.asyncio +async def test_post_with_data(server): + url = "http://127.0.0.1:8000/echo_form_data" + response = await requests_async.post(url, data={"a": "b"}) + assert response.status_code == 200 + assert response.json() == {"method": "POST", "url": url, "form": {"a": "b"}} + + +@pytest.mark.asyncio +async def test_put_with_data(server): + url = "http://127.0.0.1:8000/echo_form_data" + response = await requests_async.put(url, data={"a": "b"}) + assert response.status_code == 200 + assert response.json() == {"method": "PUT", "url": url, "form": {"a": "b"}} + + +@pytest.mark.asyncio +async def test_patch_with_data(server): + url = "http://127.0.0.1:8000/echo_form_data" + response = await requests_async.patch(url, data={"a": "b"}) + assert response.status_code == 200 + assert response.json() == {"method": "PATCH", "url": url, "form": {"a": "b"}} diff --git a/tests/test_sessions.py b/tests/test_sessions.py new file mode 100644 index 0000000..62ada18 --- /dev/null +++ b/tests/test_sessions.py @@ -0,0 +1,37 @@ +import asyncio +import requests_async +import pytest + + +@pytest.mark.asyncio +async def test_session(server): + url = "http://127.0.0.1:8000/" + + with requests_async.Session() as session: + response = await session.get(url) + assert response.status_code == 200 + assert response.json() == {"method": "GET", "url": url, "body": ""} + + response = await session.post(url) + assert response.status_code == 200 + assert response.json() == {"method": "POST", "url": url, "body": ""} + + response = await session.put(url) + assert response.status_code == 200 + assert response.json() == {"method": "PUT", "url": url, "body": ""} + + response = await session.patch(url) + assert response.status_code == 200 + assert response.json() == {"method": "PATCH", "url": url, "body": ""} + + response = await session.delete(url) + assert response.status_code == 200 + assert response.json() == {"method": "DELETE", "url": url, "body": ""} + + response = await session.options(url) + assert response.status_code == 200 + assert response.json() == {"method": "OPTIONS", "url": url, "body": ""} + + response = await session.head(url) + assert response.status_code == 200 + assert response.text == ""