diff --git a/requests_html.py b/requests_html.py index 5bb1768..e2a66f1 100644 --- a/requests_html.py +++ b/requests_html.py @@ -485,6 +485,17 @@ class HTML(BaseParser): def __next__(self): return self._next(fetch=True, next_symbol=self.next_symbol).html + def __aiter__(self): + return self + + async def __anext__(self): + while True: + url = self._next(fetch=False, next_symbol=self.next_symbol) + if not url: + break + response = await self.session.get(url) + return response.html + def add_next_symbol(self, next_symbol): self.next_symbol.append(next_symbol) diff --git a/tests/test_internet.py b/tests/test_internet.py index 19527bb..8ab99b6 100644 --- a/tests/test_internet.py +++ b/tests/test_internet.py @@ -1,7 +1,9 @@ -from requests_html import HTMLSession +import pytest +from requests_html import HTMLSession, AsyncHTMLSession session = HTMLSession() + def test_pagination(): pages = ( 'https://xkcd.com/1957/', @@ -14,3 +16,17 @@ def test_pagination(): r = session.get(page) assert next(r.html) + +@pytest.mark.asyncio +async def test_pagination(event_loop): + asession = AsyncHTMLSession() + pages = ( + 'https://xkcd.com/1957/', + 'https://reddit.com/', + 'https://smile.amazon.com/', + 'https://theverge.com/archives' + ) + + for page in pages: + r = await asession.get(page) + assert await r.html.__anext__()