]> git.phdru.name Git - bookmarks_db.git/commitdiff
Feat(Robots): Robot based on `curl-cffi.request.AsyncSession()`
authorOleg Broytman <phd@phdru.name>
Thu, 6 Mar 2025 20:08:47 +0000 (23:08 +0300)
committerOleg Broytman <phd@phdru.name>
Thu, 6 Mar 2025 20:08:47 +0000 (23:08 +0300)
Robots/bkmk_rasynccurlcffi.py [new file with mode: 0644]
doc/TODO

diff --git a/Robots/bkmk_rasynccurlcffi.py b/Robots/bkmk_rasynccurlcffi.py
new file mode 100644 (file)
index 0000000..7a7287d
--- /dev/null
@@ -0,0 +1,54 @@
+"""Robot based on curl-cffi.request.AsyncSession()
+
+This file is a part of Bookmarks database and Internet robot.
+
+"""
+
+__author__ = "Oleg Broytman <phd@phdru.name>"
+__copyright__ = "Copyright (C) 2025 PhiloSoft Design"
+__license__ = "GNU GPL"
+
+__all__ = ['robot_asynccurlcffi']
+
+
+from curl_cffi.requests import AsyncSession
+import curl_cffi
+
+from Robots.aio_base import aio_base
+from Robots.util import get_ftp
+
+
+class robot_asynccurlcffi(aio_base):
+    def version_str(self):
+        return 'curl-cffi/%s' % curl_cffi.__version__
+
+    async def get(self, url, req_headers, use_proxy=False):
+        if url.startswith('ftp://'):
+            error, welcome, body = get_ftp(url, self.timeout)
+            if error is not None:
+                return error, None, None, None
+            self.welcome = welcome
+            return None, None, None, body
+
+        if use_proxy:
+            proxies = {'http': self.proxy, 'https': self.proxy}
+        else:
+            proxies = None
+
+        error = r = None
+        try:
+            async with AsyncSession() as s:
+                r = await s.get(url, headers=req_headers,
+                                timeout=self.timeout,
+                                allow_redirects=False, proxies=proxies,
+                                verify=False, impersonate='firefox133')
+        except curl_cffi.CurlError as e:
+            error = str(e)
+            return error, None, None, None
+
+        return None, r.status_code, r.headers, r.content
+
+    def get_ftp_welcome(self):
+        welcome = self.welcome
+        self.welcome = ''
+        return welcome
index bdc06753734b46d781105e72636efd380eeac331..385b0bacff9ac5be425c014f5b5c8dc556607cde 100644 (file)
--- a/doc/TODO
+++ b/doc/TODO
@@ -1,5 +1,3 @@
-Async curl-cffi.
-
 Twisted.
 
 Selenium or PlayWright.