Initial commit

.gitignore

@@ -0,0 +1,8 @@
+*.pyc
+.coverage
+.pytest_cache/
+.mypy_cache/
+__pycache__/
+htmlcov/
+*.egg-info/
+venv/

.travis.yml

@@ -0,0 +1,20 @@
+sudo: required
+dist: xenial
+language: python
+
+cache: pip
+
+python:
+    - "3.6"
+    - "3.7"
+    - "3.8-dev"
+
+install:
+    - pip install -U -r requirements.txt
+
+script:
+    - scripts/test
+
+after_script:
+    - pip install codecov
+    - codecov

LICENSE.md

@@ -0,0 +1,27 @@
+Copyright © 2019, [Encode OSS Ltd](https://www.encode.io/).
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -0,0 +1,33 @@
+# requests-async
+
+Brings support for `async`/`await` syntax to Python's fabulous `requests` library.
+
+**This is just a first-pass right now.**
+
+Next set of things to deal with:
+
+* https support, and certificate checking.
+* streaming support for uploads and downloads.
+* connection pooling.
+* async redirections.
+* async cookie persistence, for on-disk cookie stores.
+* make sure authentication works okay (does it use adapters, is the API broken there now?)
+
+Installation:
+
+```shell
+$ pip install requests-async
+```
+
+Usage:
+
+Just use the standard requests API, but use `await` for making requests.
+
+```python
+import requests_async as requests
+
+
+response = await requests.get('http://example.org')
+print(response.status_code)
+print(response.text)
+```

requests_async/__init__.py

@@ -0,0 +1,16 @@
+from .adapters import HTTPAdapter
+from .sessions import Session
+from .api import request, get, head, post, patch, put, delete, options
+
+__version__ = "0.0.1"
+__all__ = [
+    "request",
+    "get",
+    "head",
+    "post",
+    "patch",
+    "put",
+    "delete",
+    "options",
+    "Session",
+]

requests_async/adapters.py

@@ -0,0 +1,81 @@
+import asyncio
+import io
+import typing
+from urllib.parse import urlparse
+
+import h11
+import requests
+import urllib3
+
+
+class HTTPAdapter(requests.adapters.HTTPAdapter):
+    async def send(
+        self, request: requests.PreparedRequest, *args: typing.Any, **kwargs: typing.Any
+    ) -> requests.Response:
+        urlparts = urlparse(request.url)
+
+        hostname = urlparts.hostname
+        port = urlparts.port
+        if port is None:
+            port = {"http": 80, "https": 443}[urlparts.scheme]
+        target = urlparts.path
+        if urlparts.query:
+            target += "?" + urlparts.query
+        headers = [("host", urlparts.netloc)] + list(request.headers.items())
+
+        reader, writer = await asyncio.open_connection(hostname, port)
+
+        conn = h11.Connection(our_role=h11.CLIENT)
+
+        message = h11.Request(method=request.method, target=target, headers=headers)
+        data = conn.send(message)
+        writer.write(data)
+
+        if request.body:
+            message = h11.Data(data=request.body.encode("utf-8"))
+            data = conn.send(message)
+            writer.write(data)
+
+        message = h11.EndOfMessage()
+        data = conn.send(message)
+        writer.write(data)
+
+        status_code = 0
+        headers = []
+        reason = b""
+        buffer = io.BytesIO()
+
+        while True:
+            event = conn.next_event()
+            event_type = type(event)
+
+            if event_type is h11.NEED_DATA:
+                data = await reader.read(2048)
+                conn.receive_data(data)
+
+            elif event_type is h11.Response:
+                status_code = event.status_code
+                headers = [
+                    (key.decode(), value.decode()) for key, value in event.headers
+                ]
+                reason = event.reason
+
+            elif event_type is h11.Data:
+                buffer.write(event.data)
+
+            elif event_type is h11.EndOfMessage:
+                buffer.seek(0)
+                break
+
+        writer.close()
+        await writer.wait_closed()
+
+        resp = urllib3.HTTPResponse(
+            body=buffer,
+            headers=headers,
+            status=status_code,
+            reason=reason,
+            preload_content=False,
+        )
+
+        return self.build_response(request, resp)

requests_async/api.py

@@ -0,0 +1,37 @@
+from . import sessions
+
+
+async def request(method, url, **kwargs):
+    with sessions.Session() as session:
+        return await session.request(method=method, url=url, **kwargs)
+
+
+async def get(url, params=None, **kwargs):
+    kwargs.setdefault("allow_redirects", True)
+    return await request("get", url, params=params, **kwargs)
+
+
+async def options(url, **kwargs):
+    kwargs.setdefault("allow_redirects", True)
+    return await request("options", url, **kwargs)
+
+
+async def head(url, **kwargs):
+    kwargs.setdefault("allow_redirects", False)
+    return await request("head", url, **kwargs)
+
+
+async def post(url, data=None, json=None, **kwargs):
+    return await request("post", url, data=data, json=json, **kwargs)
+
+
+async def put(url, data=None, **kwargs):
+    return await request("put", url, data=data, **kwargs)
+
+
+async def patch(url, data=None, **kwargs):
+    return await request("patch", url, data=data, **kwargs)
+
+
+async def delete(url, **kwargs):
+    return await request("delete", url, **kwargs)

requests_async/sessions.py

@@ -0,0 +1,159 @@
+import datetime
+import requests
+from . import adapters
+
+
+class Session(requests.Session):
+    def __init__(self, *args, **kwargs) -> None:
+        super(Session, self).__init__(*args, **kwargs)
+        adapter = adapters.HTTPAdapter()
+        self.mount("http://", adapter)
+        self.mount("https://", adapter)
+
+    async def request(
+        self,
+        method,
+        url,
+        params=None,
+        data=None,
+        headers=None,
+        cookies=None,
+        files=None,
+        auth=None,
+        timeout=None,
+        allow_redirects=True,
+        proxies=None,
+        hooks=None,
+        stream=None,
+        verify=None,
+        cert=None,
+        json=None,
+    ):
+        # Create the Request.
+        req = requests.models.Request(
+            method=method.upper(),
+            url=url,
+            headers=headers,
+            files=files,
+            data=data or {},
+            json=json,
+            params=params or {},
+            auth=auth,
+            cookies=cookies,
+            hooks=hooks,
+        )
+        prep = self.prepare_request(req)
+
+        proxies = proxies or {}
+
+        settings = self.merge_environment_settings(
+            prep.url, proxies, stream, verify, cert
+        )
+
+        # Send the request.
+        send_kwargs = {"timeout": timeout, "allow_redirects": allow_redirects}
+        send_kwargs.update(settings)
+        resp = await self.send(prep, **send_kwargs)
+
+        return resp
+
+    async def get(self, url, **kwargs):
+        kwargs.setdefault("allow_redirects", True)
+        return await self.request("GET", url, **kwargs)
+
+    async def options(self, url, **kwargs):
+        kwargs.setdefault("allow_redirects", True)
+        return await self.request("OPTIONS", url, **kwargs)
+
+    async def head(self, url, **kwargs):
+        kwargs.setdefault("allow_redirects", False)
+        return await self.request("HEAD", url, **kwargs)
+
+    async def post(self, url, data=None, json=None, **kwargs):
+        return await self.request("POST", url, data=data, json=json, **kwargs)
+
+    async def put(self, url, data=None, **kwargs):
+        return await self.request("PUT", url, data=data, **kwargs)
+
+    async def patch(self, url, data=None, **kwargs):
+        return await self.request("PATCH", url, data=data, **kwargs)
+
+    async def delete(self, url, **kwargs):
+        return await self.request("DELETE", url, **kwargs)
+
+    async def send(self, request, **kwargs):
+        """Send a given PreparedRequest.
+
+        :rtype: requests.Response
+        """
+        # Set defaults that the hooks can utilize to ensure they always have
+        # the correct parameters to reproduce the previous request.
+        kwargs.setdefault("stream", self.stream)
+        kwargs.setdefault("verify", self.verify)
+        kwargs.setdefault("cert", self.cert)
+        kwargs.setdefault("proxies", self.proxies)
+
+        # It's possible that users might accidentally send a Request object.
+        # Guard against that specific failure case.
+        if isinstance(request, requests.models.Request):
+            raise ValueError("You can only send PreparedRequests.")
+
+        # Set up variables needed for resolve_redirects and dispatching of hooks
+        allow_redirects = kwargs.pop("allow_redirects", True)
+        stream = kwargs.get("stream")
+        hooks = request.hooks
+
+        # Get the appropriate adapter to use
+        adapter = self.get_adapter(url=request.url)
+
+        # Start time (approximately) of the request
+        start = requests.sessions.preferred_clock()
+
+        # Send the request
+        r = await adapter.send(request, **kwargs)
+
+        # Total elapsed time of the request (approximately)
+        elapsed = requests.sessions.preferred_clock() - start
+        r.elapsed = datetime.timedelta(seconds=elapsed)
+
+        # Response manipulation hooks
+        r = requests.hooks.dispatch_hook("response", hooks, r, **kwargs)
+
+        # Persist cookies
+        if r.history:
+
+            # If the hooks create history then we want those cookies too
+            for resp in r.history:
+                requests.cookies.extract_cookies_to_jar(
+                    self.cookies, resp.request, resp.raw
+                )
+
+        requests.cookies.extract_cookies_to_jar(self.cookies, request, r.raw)
+
+        # Redirect resolving generator.
+        gen = self.resolve_redirects(r, request, **kwargs)
+
+        # Resolve redirects if allowed.
+        history = [resp for resp in gen] if allow_redirects else []
+
+        # Shuffle things around if there's history.
+        if history:
+            # Insert the first (original) request at the start
+            history.insert(0, r)
+            # Get the last request made
+            r = history.pop()
+            r.history = history
+
+        # If redirects aren't being followed, store the response on the Request for Response.next().
+        if not allow_redirects:
+            try:
+                r._next = next(
+                    self.resolve_redirects(r, request, yield_requests=True, **kwargs)
+                )
+            except StopIteration:
+                pass
+
+        if not stream:
+            r.content
+
+        return r

requirements.txt

@@ -0,0 +1,12 @@
+h11
+requests
+
+# Testing
+black
+isort
+pytest
+pytest-asyncio
+pytest-cov
+python-multipart
+starlette
+uvicorn

scripts/clean

@@ -0,0 +1,14 @@
+#!/bin/sh -e
+
+if [ -d 'dist' ] ; then
+    rm -r dist
+fi
+if [ -d 'site' ] ; then
+    rm -r site
+fi
+if [ -d 'htmlcov' ] ; then
+    rm -r htmlcov
+fi
+if [ -d 'requests_async.egg-info' ] ; then
+    rm -r requests_async.egg-info
+fi

scripts/lint

@@ -0,0 +1,12 @@
+#!/bin/sh -e
+
+export PACKAGE="requests_async"
+export PREFIX=""
+if [ -d 'venv' ] ; then
+    export PREFIX="venv/bin/"
+fi
+
+set -x
+
+${PREFIX}black ${PACKAGE} tests setup.py
+${PREFIX}isort --multi-line=3 --trailing-comma --force-grid-wrap=0 --combine-as --line-width 88 --recursive --apply ${PACKAGE} tests setup.py

scripts/publish

@@ -0,0 +1,28 @@
+#!/bin/sh -e
+
+export PACKAGE="requests_async"
+export VERSION=`cat ${PACKAGE}/__init__.py | grep __version__ | sed "s/__version__ = //" | sed "s/'//g"`
+export PREFIX=""
+if [ -d 'venv' ] ; then
+    export PREFIX="venv/bin/"
+fi
+
+scripts/clean
+
+if ! command -v "${PREFIX}twine" &>/dev/null ; then
+    echo "Unable to find the 'twine' command."
+    echo "Install from PyPI, using '${PREFIX}pip install twine'."
+    exit 1
+fi
+
+find ${PACKAGE} -type f -name "*.py[co]" -delete
+find ${PACKAGE} -type d -name __pycache__ -delete
+
+${PREFIX}python setup.py sdist
+${PREFIX}twine upload dist/*
+
+echo "You probably want to also tag the version now:"
+echo "git tag -a ${VERSION} -m 'version ${VERSION}'"
+echo "git push --tags"
+
+scripts/clean

scripts/test

@@ -0,0 +1,12 @@
+#!/bin/sh -e
+
+export PACKAGE="requests_async"
+export PREFIX=""
+if [ -d 'venv' ] ; then
+    export PREFIX="venv/bin/"
+fi
+
+set -x
+
+PYTHONPATH=. ${PREFIX}pytest --ignore venv --cov tests --cov ${PACKAGE} --cov-report= ${@}
+${PREFIX}coverage report

setup.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import re
+
+from setuptools import setup
+
+
+def get_version(package):
+    """
+    Return package version as listed in `__version__` in `init.py`.
+    """
+    with open(os.path.join(package, "__init__.py")) as f:
+        return re.search("__version__ = ['\"]([^'\"]+)['\"]", f.read()).group(1)
+
+
+def get_long_description():
+    """
+    Return the README.
+    """
+    with open("README.md", encoding="utf8") as f:
+        return f.read()
+
+
+def get_packages(package):
+    """
+    Return root package and all sub-packages.
+    """
+    return [
+        dirpath
+        for dirpath, dirnames, filenames in os.walk(package)
+        if os.path.exists(os.path.join(dirpath, "__init__.py"))
+    ]
+
+
+setup(
+    name="requests-async",
+    python_requires=">=3.6",
+    version=get_version("requests_async"),
+    url="https://github.com/encode/requests-async",
+    license="BSD",
+    description="async-await support for `requests`.",
+    long_description=get_long_description(),
+    long_description_content_type="text/markdown",
+    author="Tom Christie",
+    author_email="tom@tomchristie.com",
+    packages=get_packages("requests_async"),
+    data_files=[("", ["LICENSE.md"])],
+    install_requires=["requests", "h11"],
+    classifiers=[
+        "Development Status :: 3 - Alpha",
+        "Environment :: Web Environment",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: BSD License",
+        "Operating System :: OS Independent",
+        "Topic :: Internet :: WWW/HTTP",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+    ],
+)

tests/conftest.py

@@ -0,0 +1,52 @@
+import asyncio
+import pytest
+
+from starlette.applications import Starlette
+from starlette.responses import JSONResponse
+from starlette.routing import Route
+
+from uvicorn.config import Config
+from uvicorn.main import Server
+
+
+async def echo_request(request):
+    body = await request.body()
+    return JSONResponse(
+        {
+            "method": request.method,
+            "url": str(request.url),
+            "body": body.decode("utf-8"),
+        }
+    )
+
+
+async def echo_form_data(request):
+    form = await request.form()
+    return JSONResponse(
+        {
+            "method": request.method,
+            "url": str(request.url),
+            "form": {key: value for key, value in form.items()},
+        }
+    )
+
+
+routes = [
+    Route("/", echo_request, methods=["GET", "DELETE", "OPTIONS", "POST", "PUT", "PATCH"]),
+    Route("/echo_form_data", echo_form_data, methods=["POST", "PUT", "PATCH"]),
+]
+
+app = Starlette(routes=routes)
+
+
+@pytest.fixture
+async def server():
+    config = Config(app=app, lifespan="off")
+    server = Server(config=config)
+    task = asyncio.create_task(server.serve())
+    try:
+        while not server.started:
+            await asyncio.sleep(0.0001)
+        yield server
+    finally:
+        task.cancel()

tests/test_api.py

@@ -0,0 +1,75 @@
+import asyncio
+import requests_async
+import pytest
+
+
+@pytest.mark.asyncio
+async def test_get(server):
+    url = "http://127.0.0.1:8000/"
+    response = await requests_async.get(url)
+    assert response.status_code == 200
+    assert response.json() == {"method": "GET", "url": url, "body": ""}
+
+
+@pytest.mark.asyncio
+async def test_get_queryparams(server):
+    url = "http://127.0.0.1:8000/"
+    response = await requests_async.get(url, params={"a": "b"})
+    assert response.status_code == 200
+    assert response.json() == {"method": "GET", "url": url + "?a=b", "body": ""}
+
+
+@pytest.mark.asyncio
+async def test_head(server):
+    url = "http://127.0.0.1:8000/"
+    response = await requests_async.head(url)
+    assert response.status_code == 200
+    assert response.text == ""
+
+
+@pytest.mark.asyncio
+async def test_options(server):
+    url = "http://127.0.0.1:8000/"
+    response = await requests_async.options(url)
+    assert response.status_code == 200
+    assert response.json() == {"method": "OPTIONS", "url": url, "body": ""}
+
+
+@pytest.mark.asyncio
+async def test_delete(server):
+    url = "http://127.0.0.1:8000/"
+    response = await requests_async.delete(url)
+    assert response.status_code == 200
+    assert response.json() == {"method": "DELETE", "url": url, "body": ""}
+
+
+@pytest.mark.asyncio
+async def test_post(server):
+    url = "http://127.0.0.1:8000/echo_form_data"
+    response = await requests_async.post(url)
+    assert response.status_code == 200
+    assert response.json() == {"method": "POST", "url": url, "form": {}}
+
+
+@pytest.mark.asyncio
+async def test_post_with_data(server):
+    url = "http://127.0.0.1:8000/echo_form_data"
+    response = await requests_async.post(url, data={"a": "b"})
+    assert response.status_code == 200
+    assert response.json() == {"method": "POST", "url": url, "form": {"a": "b"}}
+
+
+@pytest.mark.asyncio
+async def test_put_with_data(server):
+    url = "http://127.0.0.1:8000/echo_form_data"
+    response = await requests_async.put(url, data={"a": "b"})
+    assert response.status_code == 200
+    assert response.json() == {"method": "PUT", "url": url, "form": {"a": "b"}}
+
+
+@pytest.mark.asyncio
+async def test_patch_with_data(server):
+    url = "http://127.0.0.1:8000/echo_form_data"
+    response = await requests_async.patch(url, data={"a": "b"})
+    assert response.status_code == 200
+    assert response.json() == {"method": "PATCH", "url": url, "form": {"a": "b"}}

tests/test_sessions.py

@@ -0,0 +1,37 @@
+import asyncio
+import requests_async
+import pytest
+
+
+@pytest.mark.asyncio
+async def test_session(server):
+    url = "http://127.0.0.1:8000/"
+
+    with requests_async.Session() as session:
+        response = await session.get(url)
+        assert response.status_code == 200
+        assert response.json() == {"method": "GET", "url": url, "body": ""}
+
+        response = await session.post(url)
+        assert response.status_code == 200
+        assert response.json() == {"method": "POST", "url": url, "body": ""}
+
+        response = await session.put(url)
+        assert response.status_code == 200
+        assert response.json() == {"method": "PUT", "url": url, "body": ""}
+
+        response = await session.patch(url)
+        assert response.status_code == 200
+        assert response.json() == {"method": "PATCH", "url": url, "body": ""}
+
+        response = await session.delete(url)
+        assert response.status_code == 200
+        assert response.json() == {"method": "DELETE", "url": url, "body": ""}
+
+        response = await session.options(url)
+        assert response.status_code == 200
+        assert response.json() == {"method": "OPTIONS", "url": url, "body": ""}
+
+        response = await session.head(url)
+        assert response.status_code == 200
+        assert response.text == ""