Feat(Robots): Make all robots async

Split check_bookmark() into sync and async variants.

diff --git a/Robots/base.py b/Robots/base.py
index c63504bad530eb2d713ceaf9171c81b44fc2decb..88f924b45d5fab07a33a218b08f3586fe59a2643 100644 (file)
--- a/Robots/base.py
+++ b/Robots/base.py
@@ -13,6 +13,7 @@ __all__ = ['robot_base']
 
 from base64 import b64encode
 from urllib.parse import urlsplit, urljoin
+import asyncio
 import sys
 import socket
 import time
@@ -93,12 +94,15 @@ class robot_base(Robot):
         return self.__class__.__name__
 
     def check_bookmark(self, bookmark):
+        return asyncio.run(self.check_bookmark_async(bookmark))
+
+    async def check_bookmark_async(self, bookmark):
         try:
             self.start = int(time.time())
             bookmark.icon = None
 
             error, http_status_code, redirect_to, headers, content = \
-                self.get_url(bookmark, bookmark.href, True)
+                await self.get_url(bookmark, bookmark.href, True)
 
             if error is not None:
                 bookmark.error = error
@@ -213,7 +217,7 @@ class robot_base(Robot):
                                     icon_error, \
                                         icon_status_code, icon_redirect_to, \
                                         icon_headers, icon_data = \
-                                        self.get_url(bookmark, _icon_url)
+                                        await self.get_url(bookmark, _icon_url)
                                     if icon_error:
                                         raise IOError("No icon: " + icon_error)
                                         break
@@ -337,7 +341,7 @@ class robot_base(Robot):
         finally:
             self.finish_check_url(bookmark)
 
-    def get_url(self, bookmark, url, accept_charset=False):
+    async def get_url(self, bookmark, url, accept_charset=False):
         split_results = urlsplit(url)
         url_proto = split_results.scheme
         url_host = split_results.hostname
@@ -359,13 +363,13 @@ class robot_base(Robot):
         if use_proxy and url_host in self.proxy_ok:
             self.log('   Immediately trying with the proxy')
             error, http_status_code, redirect_to, headers, content = \
-                self.get(bookmark, url,
-                         accept_charset=accept_charset,
-                         use_proxy=True)
+                await self.get(bookmark, url,
+                               accept_charset=accept_charset,
+                               use_proxy=True)
         else:
             error, http_status_code, redirect_to, headers, content = \
-                self.get(bookmark, url,
-                         accept_charset=accept_charset)
+                await self.get(bookmark, url,
+                               accept_charset=accept_charset)
             if error is not None and (
                 not url_host.startswith('localhost') and
                 not url_host.startswith('127.')
@@ -374,9 +378,9 @@ class robot_base(Robot):
                 if use_proxy and http_status_code != 404:
                     self.log('   Retrying with the proxy...')
                     error, http_status_code, redirect_to, headers, content = \
-                        self.get(bookmark, url,
-                                 accept_charset=accept_charset,
-                                 use_proxy=True)
+                        await self.get(bookmark, url,
+                                       accept_charset=accept_charset,
+                                       use_proxy=True)
                     if error is None:
                         self.proxy_ok.add(url_host)
         if (error is not None) or (

diff --git a/Robots/bkmk_raiohttp.py b/Robots/bkmk_raiohttp.py
index 22c84d2d74e453b0c688b81f4bb5dba7b42327dc..b195ff28f830ffb1b07c6d375f723ba25eb38822 100644 (file)
--- a/Robots/bkmk_raiohttp.py
+++ b/Robots/bkmk_raiohttp.py
@@ -27,11 +27,11 @@ class robot_aiohttp(robot_base):
     def version_str(self):
         return 'aiohttp/%s' % aiohttp.__version__
 
-    def get(self, bookmark, url, accept_charset=False, use_proxy=False):
+    async def get(self, bookmark, url, accept_charset=False, use_proxy=False):
         if url.startswith('ftp://'):
-            error, body = asyncio.run(_get_ftp(
+            error, body = await _get_ftp(
                 url, timeout=self.ftp_timeout,
-            ))
+            )
             if error is not None:
                 error = str(error)
                 return error, None, None, None, None
@@ -48,10 +48,10 @@ class robot_aiohttp(robot_base):
         else:
             proxy = None
 
-        error, status, resp_headers, body = asyncio.run(_get_http(
+        error, status, resp_headers, body = await _get_http(
             url, headers=headers, proxy=proxy,
             timeout=self.timeout,
-        ))
+        )
         if error is not None or (status and status >= 400):
             if error is None:
                 error = 'Error %d' % status

diff --git a/Robots/bkmk_rcurl.py b/Robots/bkmk_rcurl.py
index 6c50c30e1d989fbf766811ecf62cf55872780a45..9f18d8c4432ad40c4c2e0845892c7f889e0b0d3e 100644 (file)
--- a/Robots/bkmk_rcurl.py
+++ b/Robots/bkmk_rcurl.py
@@ -24,7 +24,7 @@ class robot_curl(robot_base):
     def version_str(self):
         return str(pycurl.version)
 
-    def get(self, bookmark, url, accept_charset=False, use_proxy=False):
+    async def get(self, bookmark, url, accept_charset=False, use_proxy=False):
         if accept_charset and bookmark.charset:
             headers = request_headers.copy()
             headers['Accept-Charset'] = bookmark.charset

diff --git a/Robots/bkmk_rrequests.py b/Robots/bkmk_rrequests.py
index f1e5ba1dd1ac35fff864acd27f8d2e3a9914d2f1..2fd83319d12ea84bcda19da271f5d5956bb1a8e3 100644 (file)
--- a/Robots/bkmk_rrequests.py
+++ b/Robots/bkmk_rrequests.py
@@ -28,7 +28,7 @@ class robot_requests(robot_base):
     def version_str(self):
         return 'python-requests urllib3/%s' % urllib3.__version__
 
-    def get(self, bookmark, url, accept_charset=False, use_proxy=False):
+    async def get(self, bookmark, url, accept_charset=False, use_proxy=False):
         if url.startswith('ftp://'):
             error, welcome, body = _get_ftp(url, self.timeout)
             if error is not None:

diff --git a/doc/ANNOUNCE b/doc/ANNOUNCE
index 82765aaff8c2f9c22210a97cd91e7397de5c18c2..47b45a6f29cfd26261152835a4273c0f77d6eaac 100644 (file)
--- a/doc/ANNOUNCE
+++ b/doc/ANNOUNCE
@@ -9,6 +9,9 @@ WHAT'S NEW
 
 Version 6.0.0 (2024-??-??)
 
+   Make all robots async.
+   Split check_bookmark() into sync and async variants.
+
    Renamed max_workers to max_urls.
 
 
@@ -17,7 +20,7 @@ WHERE TO GET
      git clone https://git.phdru.name/bookmarks_db.git
      git clone  git://git.phdru.name/bookmarks_db.git
 
-   Requires: Python 3.6+, BeautifulSoup 4, lxml, m_lib 2.0+,
+   Requires: Python 3.7+, BeautifulSoup 4, lxml, m_lib 2.0+,
    requests and requests-ftp.
 
 

diff --git a/doc/ChangeLog b/doc/ChangeLog
index 32d24d81d6fa8f2fa0394c1955628a592e0b7c35..5e08408a24589b398a3956a4c7d5e62798d88752 100644 (file)
--- a/doc/ChangeLog
+++ b/doc/ChangeLog
@@ -1,5 +1,8 @@
 Version 6.0.0 (2024-??-??)
 
+   Make all robots async.
+   Split check_bookmark() into sync and async variants.
+
    Renamed max_workers to max_urls.
 
 Version 5.7.0 (2024-08-16)

diff --git a/setup.py b/setup.py
index b789bcb4a2605050211f2da4df021cfb1419c766..cba7fe2444a1719a032b0be5d1877af3ea90c9a2 100755 (executable)
--- a/setup.py
+++ b/setup.py
@@ -23,7 +23,6 @@ setup(
         'Operating System :: OS Independent',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3 :: Only',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
@@ -31,7 +30,7 @@ setup(
         'Programming Language :: Python :: 3.11',
         'Programming Language :: Python :: 3.12',
     ],
-    python_requires='>=3.6',
+    python_requires='>=3.7',
     install_requires=[
         'm_lib.full>=1.0',
     ],