From aae5f1367c78115113742372b1d8bdd51e377bdc Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 28 Feb 2018 07:12:14 -0500 Subject: [PATCH 1/8] better docstrings Signed-off-by: Kenneth Reitz --- requests_html.py | 37 +++++++++++++++++++++++++++++++++---- setup.py | 2 +- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/requests_html.py b/requests_html.py index d4ba93f..92b8b40 100644 --- a/requests_html.py +++ b/requests_html.py @@ -24,7 +24,14 @@ useragent = UserAgent() class BaseParser: - """A basic HTML/Element Parser, for Humans.""" + """A basic HTML/Element Parser, for Humans. + + :param element: The element from which to base the parsing upon. + :param default_encoding: Which encoding to default to. + :param html: HTML from which to base the parsing upon (optional). + :param url: The URL from which the HTML originated, used for ``absolute_links``. + + """ def __init__(self, *, element, default_encoding: str = None, html: str = None, url: str) -> None: self.element = element @@ -96,6 +103,9 @@ class BaseParser: def find(self, selector: str, first: bool = False, _encoding: str = None): """Given a CSS Selector, returns a list of :class:`Element ` objects. + :param selector: CSS Selector to use. + :param first: Whether or not to return just the first result. + Example CSS Selectors: - ``a`` @@ -125,6 +135,9 @@ class BaseParser: """Given an XPath selector, returns a list of :class:`Element ` objects. + :param selector: XPath Selector to use. + :param first: Whether or not to return just the first result. + If a sub-selector is specified (e.g. ``//a/@href``), a simple list of results is returned. @@ -153,12 +166,18 @@ class BaseParser: return c def search(self, template: str) -> Result: - """Searches the :class:`Element ` for the given parse template.""" + """Searches the :class:`Element ` for the given Parse template. + + :param template: The Parse template to use. + """ + return parse_search(template, self.html) def search_all(self, template: str) -> Result: """Searches the :class:`Element ` (multiple times) for the given parse template. + + :param template: The Parse template to use. """ return [r for r in findall(template, self.html)] @@ -221,7 +240,12 @@ class BaseParser: class Element(BaseParser): - """An element of HTML.""" + """An element of HTML. + + :param element: The element from which to base the parsing upon. + :param url: The URL from which the HTML originated, used for ``absolute_links``. + :param default_encoding: Which encoding to default to. + """ def __init__(self, *, element, url, default_encoding) -> None: super(Element, self).__init__(element=element, url=url, default_encoding=default_encoding) @@ -249,7 +273,12 @@ class Element(BaseParser): class HTML(BaseParser): - """An HTML document, ready for parsing.""" + """An HTML document, ready for parsing. + + :param url: The URL from which the HTML originated, used for ``absolute_links``. + :param html: HTML from which to base the parsing upon (optional). + :param default_encoding: Which encoding to default to. + """ def __init__(self, *, url=DEFAULT_URL, html, default_encoding=DEFAULT_ENCODING) -> None: diff --git a/setup.py b/setup.py index cefc1fb..4350866 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ DESCRIPTION = 'HTML Parsing for Humans.' URL = 'https://github.com/kennethreitz/requests-html' EMAIL = 'me@kennethreitz.org' AUTHOR = 'Kenneth Reitz' -VERSION = '0.6.5' +VERSION = '0.6.6' # What packages are required for this module to be executed? REQUIRED = [ From 77c690629256613fac2bd767c9cc05db38d744f9 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 28 Feb 2018 07:28:35 -0500 Subject: [PATCH 2/8] mypy stuff Signed-off-by: Kenneth Reitz --- requests_html.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/requests_html.py b/requests_html.py index 92b8b40..71ce376 100644 --- a/requests_html.py +++ b/requests_html.py @@ -1,7 +1,7 @@ import asyncio from urllib.parse import urlparse, urlunparse from concurrent.futures._base import TimeoutError -from typing import Set +from typing import Set, Union, List import pyppeteer import requests @@ -21,6 +21,10 @@ DEFAULT_URL = 'https://example.org/' useragent = UserAgent() +# Typing. +_Find = Union[List['Element'], 'Element'] +_XPath = Union[List[str], List['Element'], str, 'Element'] +_HTML = Union[str, bytes] class BaseParser: @@ -33,12 +37,17 @@ class BaseParser: """ - def __init__(self, *, element, default_encoding: str = None, html: str = None, url: str) -> None: + def __init__(self, *, element, default_encoding: str = None, html: _HTML = None, url: str) -> None: self.element = element self.url = url self.skip_anchors = True self.default_encoding = default_encoding self._encoding = None + + # Encode incoming unicode HTML into bytes. + if isinstance(html, str): + html = html.encode(DEFAULT_ENCODING) + self._html = html @property @@ -100,7 +109,7 @@ class BaseParser: """The full text content (including links) of the :class:`Element ` or :class:`HTML `..""" return self.lxml.text_content() - def find(self, selector: str, first: bool = False, _encoding: str = None): + def find(self, selector: str, first: bool = False, _encoding: str = None) -> _Find: """Given a CSS Selector, returns a list of :class:`Element ` objects. :param selector: CSS Selector to use. @@ -131,7 +140,7 @@ class BaseParser: else: return elements - def xpath(self, selector: str, first: bool = False, _encoding: str = None): + def xpath(self, selector: str, first: bool = False, _encoding: str = None) -> _XPath: """Given an XPath selector, returns a list of :class:`Element ` objects. @@ -154,7 +163,7 @@ class BaseParser: if not isinstance(selection, etree._ElementUnicodeResult): element = Element(element=selection, url=self.url, default_encoding=_encoding or self.encoding) else: - element = selection + element = str(selection) c.append(element) if first: @@ -280,7 +289,7 @@ class HTML(BaseParser): :param default_encoding: Which encoding to default to. """ - def __init__(self, *, url=DEFAULT_URL, html, default_encoding=DEFAULT_ENCODING) -> None: + def __init__(self, *, url: str = DEFAULT_URL, html: _HTML, default_encoding: str =DEFAULT_ENCODING) -> None: # Convert incoming unicode HTML into bytes. if isinstance(html, str): @@ -382,9 +391,9 @@ class HTMLResponse(requests.Response): intelligent ``.html`` property added. """ - def __init__(self, *args, **kwargs) -> None: - super(HTMLResponse, self).__init__(*args, **kwargs) - self._html = None + def __init__(self) -> None: + super(HTMLResponse, self).__init__() + self._html = None # type: HTML @property def html(self) -> HTML: From 9a008a2ee5ac903f799325d6281bda5333b5bd51 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 28 Feb 2018 07:43:18 -0500 Subject: [PATCH 3/8] improved docs and mypy Signed-off-by: Kenneth Reitz --- requests_html.py | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/requests_html.py b/requests_html.py index 71ce376..5eede6d 100644 --- a/requests_html.py +++ b/requests_html.py @@ -1,7 +1,7 @@ import asyncio from urllib.parse import urlparse, urlunparse from concurrent.futures._base import TimeoutError -from typing import Set, Union, List +from typing import Set, Union, List, MutableMapping import pyppeteer import requests @@ -25,6 +25,17 @@ useragent = UserAgent() _Find = Union[List['Element'], 'Element'] _XPath = Union[List[str], List['Element'], str, 'Element'] _HTML = Union[str, bytes] +_BaseHTML = str +_UserAgent = str +_DefaultEncoding = str +_URL = str +_RawHTML = bytes +_Encoding = str +_LXML = HtmlElement +_Text = str +_Search = Result +_Links = Set[str] +_Attrs = MutableMapping class BaseParser: @@ -37,7 +48,7 @@ class BaseParser: """ - def __init__(self, *, element, default_encoding: str = None, html: _HTML = None, url: str) -> None: + def __init__(self, *, element, default_encoding: _DefaultEncoding = None, html: _HTML = None, url: _URL) -> None: self.element = element self.url = url self.skip_anchors = True @@ -51,7 +62,7 @@ class BaseParser: self._html = html @property - def raw_html(self) -> bytes: + def raw_html(self) -> _RawHTML: """Bytes representation of the HTML content (`learn more `_).""" if self._html: return self._html @@ -59,7 +70,7 @@ class BaseParser: return etree.tostring(self.element, encoding='unicode').strip().encode(self.encoding) @property - def html(self) -> str: + def html(self) -> _BaseHTML: """Unicode representation of the HTML content (`learn more `_).""" if self._html: return self._html.decode(self.encoding) @@ -72,7 +83,7 @@ class BaseParser: self._html = html @property - def encoding(self) -> str: + def encoding(self) -> _Encoding: """The encoding string to be used, extracted from the HTML and :class:`HTMLResponse ` headers. """ @@ -100,12 +111,12 @@ class BaseParser: return fromstring(self.html) @property - def text(self) -> str: + def text(self) -> _Text: """The text content of the :class:`Element ` or :class:`HTML `.""" return self.pq.text() @property - def full_text(self) -> str: + def full_text(self) -> _Text: """The full text content (including links) of the :class:`Element ` or :class:`HTML `..""" return self.lxml.text_content() @@ -191,7 +202,7 @@ class BaseParser: return [r for r in findall(template, self.html)] @property - def links(self) -> Set[str]: + def links(self) -> _Links: """All found links on page, in as–is form.""" def gen(): for link in self.find('a'): @@ -207,7 +218,7 @@ class BaseParser: return set(gen()) @property - def absolute_links(self) -> Set[str]: + def absolute_links(self) -> _Links: """All found links on page, in absolute form (`learn more `_). """ @@ -231,7 +242,7 @@ class BaseParser: return set(gen()) @property - def base_url(self) -> str: + def base_url(self) -> _URL: """The base URL for the page. Supports the ```` tag (`learn more `_).""" @@ -268,7 +279,7 @@ class Element(BaseParser): return "".format(repr(self.element.tag), ' '.join(attrs)) @property - def attrs(self) -> dict: + def attrs(self) -> _Attrs: """Returns a dictionary of the attributes of the :class:`Element ` (`learn more `_). """ @@ -386,9 +397,8 @@ class HTML(BaseParser): class HTMLResponse(requests.Response): - """An HTML-enabled :class:`Response ` object. - Same as Requests class:`Response ` object, but with an - intelligent ``.html`` property added. + """An HTML-enabled :class:`requests.Response ` object. + Effectively the same, but with an intelligent ``.html`` property added. """ def __init__(self) -> None: @@ -410,7 +420,7 @@ class HTMLResponse(requests.Response): return html_r -def user_agent(style='chrome') -> str: +def user_agent(style='chrome') -> _UserAgent: """Returns a random user-agent, if not requested one of a specific style. Defaults to a Chrome-style User-Agent. """ @@ -437,8 +447,8 @@ class HTMLSession(requests.Session): @staticmethod def _handle_response(response, **kwargs) -> HTMLResponse: - """Requests HTTP Response handler. Attaches .html property to Response - objects. + """Requests HTTP Response handler. Attaches .html property to + class:`requests.Response ` objects. """ if not response.encoding: response.encoding = DEFAULT_ENCODING @@ -446,6 +456,9 @@ class HTMLSession(requests.Session): return response def request(self, *args, **kwargs) -> HTMLResponse: + """Makes an HTTP Request, with mocked User–Agent headers. + Returns a class:`HTTPResponse `. + """ # Convert Request object into HTTPRequest object. r = super(HTMLSession, self).request(*args, **kwargs) html_r = HTMLResponse._from_response(r) From 2b22c2f55d4feef0e35dd9cbf04a54a27e42c884 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 28 Feb 2018 07:50:59 -0500 Subject: [PATCH 4/8] 3.6 and up Signed-off-by: Kenneth Reitz --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 4350866..ea20a2a 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,7 @@ setup( author=AUTHOR, author_email=EMAIL, url=URL, - python_requires='>=3.5.0', + python_requires='>=3.6.0', # If your package is a single module, use this instead of 'packages': py_modules=['requests_html'], @@ -93,7 +93,6 @@ setup( # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers 'License :: OSI Approved :: MIT License', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy' From ede6f9e1d3ec75884a3dcdb7c98ee50e5aa3acdb Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 28 Feb 2018 07:53:18 -0500 Subject: [PATCH 5/8] raise an error if not on python 3.6 Signed-off-by: Kenneth Reitz --- requests_html.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/requests_html.py b/requests_html.py index 5eede6d..0be722c 100644 --- a/requests_html.py +++ b/requests_html.py @@ -1,3 +1,4 @@ +import sys import asyncio from urllib.parse import urlparse, urlunparse from concurrent.futures._base import TimeoutError @@ -37,6 +38,13 @@ _Search = Result _Links = Set[str] _Attrs = MutableMapping +# Sanity checking. +try: + assert sys.version_info.major == 3 + assert sys.version_info.minor > 5 +except AssertionError: + raise RuntimeError('Requests-HTML requires Python 3.6+!') + class BaseParser: """A basic HTML/Element Parser, for Humans. From 18da03e891eb9f843e7a00d6a4a8a36c5784598a Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 28 Feb 2018 07:54:30 -0500 Subject: [PATCH 6/8] new version Signed-off-by: Kenneth Reitz --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ea20a2a..873dabe 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ DESCRIPTION = 'HTML Parsing for Humans.' URL = 'https://github.com/kennethreitz/requests-html' EMAIL = 'me@kennethreitz.org' AUTHOR = 'Kenneth Reitz' -VERSION = '0.6.6' +VERSION = '0.6.7' # What packages are required for this module to be executed? REQUIRED = [ From 2504efb35fa7c6262de5dc8d75d46d7e10e848d2 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 28 Feb 2018 08:17:46 -0500 Subject: [PATCH 7/8] support render of non-loaded websites Signed-off-by: Kenneth Reitz --- requests_html.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/requests_html.py b/requests_html.py index 0be722c..082ca5f 100644 --- a/requests_html.py +++ b/requests_html.py @@ -2,7 +2,7 @@ import sys import asyncio from urllib.parse import urlparse, urlunparse from concurrent.futures._base import TimeoutError -from typing import Set, Union, List, MutableMapping +from typing import Set, Union, List, MutableMapping, Optional import pyppeteer import requests @@ -325,7 +325,7 @@ class HTML(BaseParser): def __repr__(self) -> str: return "".format(repr(self.url)) - def render(self, retries: int = 8, script: str = None, scrolldown=False, sleep: int = 0): + def render(self, retries: int = 8, script: str = None, scrolldown=False, sleep: int = 0, reload: bool = True): """Reloads the response in Chromium, and replaces HTML content with an updated version, with JavaScript executed. @@ -361,13 +361,16 @@ class HTML(BaseParser): Warning: the first time you run this method, it will download Chromium into your home directory (``~/.pyppeteer``). """ - async def _async_render(*, url: str, script: str = None, scrolldown, sleep: int): + async def _async_render(*, url: str, script: str = None, scrolldown, sleep: int, reload: bool = True, content: Optional[str]): try: browser = pyppeteer.launch(headless=True) page = await browser.newPage() # Load the given page (GET request, obviously.) - await page.goto(url) + if reload: + await page.goto(url) + else: + await page.setContent(content) result = None if script: @@ -395,7 +398,7 @@ class HTML(BaseParser): for i in range(retries): if not content: try: - content, result = loop.run_until_complete(_async_render(url=self.url, script=script, sleep=sleep, scrolldown=scrolldown)) + content, result = loop.run_until_complete(_async_render(url=self.url, script=script, sleep=sleep, content=self.html, reload=reload, scrolldown=scrolldown)) except TimeoutError: pass From e9c162f5f76c87452fb84a833287b313ae7932a4 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Wed, 28 Feb 2018 08:20:34 -0500 Subject: [PATCH 8/8] next version Signed-off-by: Kenneth Reitz --- requests_html.py | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/requests_html.py b/requests_html.py index 082ca5f..7311667 100644 --- a/requests_html.py +++ b/requests_html.py @@ -329,6 +329,12 @@ class HTML(BaseParser): """Reloads the response in Chromium, and replaces HTML content with an updated version, with JavaScript executed. + :param retries: The number of times to retry loading the page in Chromium. + :param script: JavaScript to execute upon page load (optional). + :param scrolldown: Integer, if provided, of how many times to page down. + :param sleep: Integer, if provided, of how many long to sleep after initial render. + :param reload: If ``False``, content will not be loaded from the browser, but will be provided from memory. + If ``scrolldown`` is specified, the page will scrolldown the specified number of times, after sleeping the specified amount of time (e.g. ``scrolldown=10, sleep=1``). diff --git a/setup.py b/setup.py index 873dabe..b28f984 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ DESCRIPTION = 'HTML Parsing for Humans.' URL = 'https://github.com/kennethreitz/requests-html' EMAIL = 'me@kennethreitz.org' AUTHOR = 'Kenneth Reitz' -VERSION = '0.6.7' +VERSION = '0.6.8' # What packages are required for this module to be executed? REQUIRED = [