From 18a19897811a4ab8e27cad7f17c01eb7bec05cba Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Sun, 18 Aug 2024 23:52:21 +0300 Subject: [PATCH] Feat(Robots): Make all robots async Split check_bookmark() into sync and async variants. --- Robots/base.py | 26 +++++++++++++++----------- Robots/bkmk_raiohttp.py | 10 +++++----- Robots/bkmk_rcurl.py | 2 +- Robots/bkmk_rrequests.py | 2 +- doc/ANNOUNCE | 5 ++++- doc/ChangeLog | 3 +++ setup.py | 3 +-- 7 files changed, 30 insertions(+), 21 deletions(-) diff --git a/Robots/base.py b/Robots/base.py index c63504b..88f924b 100644 --- a/Robots/base.py +++ b/Robots/base.py @@ -13,6 +13,7 @@ __all__ = ['robot_base'] from base64 import b64encode from urllib.parse import urlsplit, urljoin +import asyncio import sys import socket import time @@ -93,12 +94,15 @@ class robot_base(Robot): return self.__class__.__name__ def check_bookmark(self, bookmark): + return asyncio.run(self.check_bookmark_async(bookmark)) + + async def check_bookmark_async(self, bookmark): try: self.start = int(time.time()) bookmark.icon = None error, http_status_code, redirect_to, headers, content = \ - self.get_url(bookmark, bookmark.href, True) + await self.get_url(bookmark, bookmark.href, True) if error is not None: bookmark.error = error @@ -213,7 +217,7 @@ class robot_base(Robot): icon_error, \ icon_status_code, icon_redirect_to, \ icon_headers, icon_data = \ - self.get_url(bookmark, _icon_url) + await self.get_url(bookmark, _icon_url) if icon_error: raise IOError("No icon: " + icon_error) break @@ -337,7 +341,7 @@ class robot_base(Robot): finally: self.finish_check_url(bookmark) - def get_url(self, bookmark, url, accept_charset=False): + async def get_url(self, bookmark, url, accept_charset=False): split_results = urlsplit(url) url_proto = split_results.scheme url_host = split_results.hostname @@ -359,13 +363,13 @@ class robot_base(Robot): if use_proxy and url_host in self.proxy_ok: self.log(' Immediately trying with the proxy') error, http_status_code, redirect_to, headers, content = \ - self.get(bookmark, url, - accept_charset=accept_charset, - use_proxy=True) + await self.get(bookmark, url, + accept_charset=accept_charset, + use_proxy=True) else: error, http_status_code, redirect_to, headers, content = \ - self.get(bookmark, url, - accept_charset=accept_charset) + await self.get(bookmark, url, + accept_charset=accept_charset) if error is not None and ( not url_host.startswith('localhost') and not url_host.startswith('127.') @@ -374,9 +378,9 @@ class robot_base(Robot): if use_proxy and http_status_code != 404: self.log(' Retrying with the proxy...') error, http_status_code, redirect_to, headers, content = \ - self.get(bookmark, url, - accept_charset=accept_charset, - use_proxy=True) + await self.get(bookmark, url, + accept_charset=accept_charset, + use_proxy=True) if error is None: self.proxy_ok.add(url_host) if (error is not None) or ( diff --git a/Robots/bkmk_raiohttp.py b/Robots/bkmk_raiohttp.py index 22c84d2..b195ff2 100644 --- a/Robots/bkmk_raiohttp.py +++ b/Robots/bkmk_raiohttp.py @@ -27,11 +27,11 @@ class robot_aiohttp(robot_base): def version_str(self): return 'aiohttp/%s' % aiohttp.__version__ - def get(self, bookmark, url, accept_charset=False, use_proxy=False): + async def get(self, bookmark, url, accept_charset=False, use_proxy=False): if url.startswith('ftp://'): - error, body = asyncio.run(_get_ftp( + error, body = await _get_ftp( url, timeout=self.ftp_timeout, - )) + ) if error is not None: error = str(error) return error, None, None, None, None @@ -48,10 +48,10 @@ class robot_aiohttp(robot_base): else: proxy = None - error, status, resp_headers, body = asyncio.run(_get_http( + error, status, resp_headers, body = await _get_http( url, headers=headers, proxy=proxy, timeout=self.timeout, - )) + ) if error is not None or (status and status >= 400): if error is None: error = 'Error %d' % status diff --git a/Robots/bkmk_rcurl.py b/Robots/bkmk_rcurl.py index 6c50c30..9f18d8c 100644 --- a/Robots/bkmk_rcurl.py +++ b/Robots/bkmk_rcurl.py @@ -24,7 +24,7 @@ class robot_curl(robot_base): def version_str(self): return str(pycurl.version) - def get(self, bookmark, url, accept_charset=False, use_proxy=False): + async def get(self, bookmark, url, accept_charset=False, use_proxy=False): if accept_charset and bookmark.charset: headers = request_headers.copy() headers['Accept-Charset'] = bookmark.charset diff --git a/Robots/bkmk_rrequests.py b/Robots/bkmk_rrequests.py index f1e5ba1..2fd8331 100644 --- a/Robots/bkmk_rrequests.py +++ b/Robots/bkmk_rrequests.py @@ -28,7 +28,7 @@ class robot_requests(robot_base): def version_str(self): return 'python-requests urllib3/%s' % urllib3.__version__ - def get(self, bookmark, url, accept_charset=False, use_proxy=False): + async def get(self, bookmark, url, accept_charset=False, use_proxy=False): if url.startswith('ftp://'): error, welcome, body = _get_ftp(url, self.timeout) if error is not None: diff --git a/doc/ANNOUNCE b/doc/ANNOUNCE index 82765aa..47b45a6 100644 --- a/doc/ANNOUNCE +++ b/doc/ANNOUNCE @@ -9,6 +9,9 @@ WHAT'S NEW Version 6.0.0 (2024-??-??) + Make all robots async. + Split check_bookmark() into sync and async variants. + Renamed max_workers to max_urls. @@ -17,7 +20,7 @@ WHERE TO GET git clone https://git.phdru.name/bookmarks_db.git git clone git://git.phdru.name/bookmarks_db.git - Requires: Python 3.6+, BeautifulSoup 4, lxml, m_lib 2.0+, + Requires: Python 3.7+, BeautifulSoup 4, lxml, m_lib 2.0+, requests and requests-ftp. diff --git a/doc/ChangeLog b/doc/ChangeLog index 32d24d8..5e08408 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,5 +1,8 @@ Version 6.0.0 (2024-??-??) + Make all robots async. + Split check_bookmark() into sync and async variants. + Renamed max_workers to max_urls. Version 5.7.0 (2024-08-16) diff --git a/setup.py b/setup.py index b789bcb..cba7fe2 100755 --- a/setup.py +++ b/setup.py @@ -23,7 +23,6 @@ setup( 'Operating System :: OS Independent', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3 :: Only', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', @@ -31,7 +30,7 @@ setup( 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', ], - python_requires='>=3.6', + python_requires='>=3.7', install_requires=[ 'm_lib.full>=1.0', ], -- 2.39.5