diff --git a/Pipfile b/Pipfile index 62550b6..76e7c31 100644 --- a/Pipfile +++ b/Pipfile @@ -24,6 +24,7 @@ pytest = "*" "e1839a8" = {path = ".", editable = true} sphinx = "*" mypy = "*" +pytest-asyncio = "*" [scripts] diff --git a/requests_html.py b/requests_html.py index 0238863..c7f6e11 100644 --- a/requests_html.py +++ b/requests_html.py @@ -1,7 +1,9 @@ import sys import asyncio from urllib.parse import urlparse, urlunparse +from concurrent.futures import ThreadPoolExecutor from concurrent.futures._base import TimeoutError +from functools import partial from typing import Set, Union, List, MutableMapping, Optional import pyppeteer @@ -599,3 +601,19 @@ class HTMLSession(requests.Session): r = super(HTMLSession, self).request(*args, **kwargs) return HTMLResponse._from_response(r) + + +class AsyncHTMLSession(requests.Session): + """ """ + + def __init__(self, *args, **kwargs): + """ Create loop and thread pool. """ + self.loop = asyncio.get_event_loop() + self.thread_pool = ThreadPoolExecutor() + + super().__init__(*args, **kwargs) + + def request(self, *args, **kwargs): + """ Partial original request func and run it in a thread. """ + func = partial(super().request, *args, **kwargs) + return self.loop.run_in_executor(self.thread_pool, func) diff --git a/tests/test_requests_html.py b/tests/test_requests_html.py index 40cc7a3..3b70215 100644 --- a/tests/test_requests_html.py +++ b/tests/test_requests_html.py @@ -1,7 +1,7 @@ import os import pytest -from requests_html import HTMLSession, HTML +from requests_html import HTMLSession, AsyncHTMLSession, HTML from requests_file import FileAdapter session = HTMLSession() @@ -15,12 +15,31 @@ def get(): return session.get(url) +@pytest.fixture +def async_get(event_loop): + """ AsyncSession cannot be created global since it will create + a different loop from pytest-asyncio. """ + async_session = AsyncHTMLSession() + async_session.mount('file://', FileAdapter()) + path = os.path.sep.join((os.path.dirname(os.path.abspath(__file__)), 'python.html')) + url = 'file://{}'.format(path) + + return async_session.get(url) + + @pytest.mark.ok def test_file_get(): r = get() assert r.status_code == 200 +@pytest.mark.ok +@pytest.mark.asyncio +async def test_async_file_get(async_get): + r = await async_get + assert r.status_code == 200 + + @pytest.mark.ok def test_class_seperation(): r = get()