Feat(bkmk_raiohttp): Use aiohttp-socks

diff --git a/Robots/bkmk_raiohttp.py b/Robots/bkmk_raiohttp.py
index 422db04d567ae8e8f541c562b72a7616120f4d6e..da01cfd4f5867a4fc9aee7ba1a549deee768009f 100644 (file)
--- a/Robots/bkmk_raiohttp.py
+++ b/Robots/bkmk_raiohttp.py
@@ -14,6 +14,8 @@ __all__ = ['robot_aiohttp']
 from urllib.parse import urlsplit
 import asyncio
 
+from aiohttp_socks import ProxyConnector
+from python_socks import parse_proxy_url
 import aioftp
 import aiohttp
 import aiohttp.client_exceptions
@@ -46,7 +48,7 @@ class robot_aiohttp(robot_base):
         else:
             proxy = None
 
-        error, status, resp_headers, body = asyncio.run(get_http(
+        error, status, resp_headers, body = asyncio.run(_get_http(
             url, headers=headers, proxy=proxy,
             timeout=self.timeout,
         ))
@@ -66,10 +68,26 @@ class robot_aiohttp(robot_base):
         return ''  # We don't store welcome message yet
 
 
-async def get_http(url, headers={}, proxy=None, timeout=60):
+async def _get_http(url, headers={}, proxy=None, timeout=60):
+    connector = None
+    if proxy and proxy.startswith('socks5'):
+        if proxy.startswith('socks5h://'):
+            proxy = proxy.replace('socks5h://', 'socks5://')
+            rdns = True
+        else:
+            rdns = False
+        proxy_type, proxy_host, proxy_port, pusername, ppassword = \
+            parse_proxy_url(proxy)
+        connector = ProxyConnector(
+            proxy_type=proxy_type, host=proxy_host, port=proxy_port,
+            username=pusername, password=ppassword, rdns=rdns,
+        )
+        proxy = None
     timeout = aiohttp.ClientTimeout(connect=timeout, total=timeout)
     try:
-        async with aiohttp.ClientSession(timeout=timeout) as session:
+        async with aiohttp.ClientSession(
+            connector=connector, timeout=timeout
+        ) as session:
             async with session.get(
                     url, headers=headers, proxy=proxy,
                     allow_redirects=False) as resp:

diff --git a/bkmk_db-venv b/bkmk_db-venv
index 25608fc3bede67b00fa74e90f246b5303b41fb27..412c901b8c675f2c531628f84e489c0c82186c91 100644 (file)
--- a/bkmk_db-venv
+++ b/bkmk_db-venv
@@ -10,6 +10,6 @@ if [ -z "$VIRTUAL_ENV" ]; then
          pip install --compile --upgrade beautifulsoup4 lxml m_lib.full \
          "requests[socks]" \
          pycurl certifi \
-         aiohttp aioftp
+         aiohttp aiohttp-socks aioftp
     }
 fi

diff --git a/doc/ANNOUNCE b/doc/ANNOUNCE
index e97f411f818bbf172b10f3a5ee0117735ecfb686..fa643b4c2584852e66222ea47a314e71825995bd 100644 (file)
--- a/doc/ANNOUNCE
+++ b/doc/ANNOUNCE
@@ -13,6 +13,8 @@ Version 5.7.0 (2024-??-??)
 
    Robots: Removed connect_timeout, added ftp_timeout.
 
+   Robot bkmk_raiohttp: Use aiohttp-socks.
+
 Version 5.6.1 (2024-08-15)
 
    Minor fixes.

diff --git a/doc/ChangeLog b/doc/ChangeLog
index 4bb123a1fb65804f037f4734d11b616d0580dea3..0fe15634c1caea6b41786ce670646058daa2a184 100644 (file)
--- a/doc/ChangeLog
+++ b/doc/ChangeLog
@@ -4,6 +4,8 @@ Version 5.7.0 (2024-??-??)
 
    Robots: Removed connect_timeout, added ftp_timeout.
 
+   Robot bkmk_raiohttp: Use aiohttp-socks.
+
 Version 5.6.1 (2024-08-15)
 
    Minor fixes.

diff --git a/setup.py b/setup.py
index 1c3160f73a94b83b0db32a63007f7940901d5b31..1e81b473ae0d2ac77dcf8b0fa7a20989eda70271 100755 (executable)
--- a/setup.py
+++ b/setup.py
@@ -22,7 +22,6 @@ setup(
         'Operating System :: OS Independent',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3 :: Only',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
@@ -32,7 +31,7 @@ setup(
         'Programming Language :: Python :: 3.11',
         'Programming Language :: Python :: 3.12',
     ],
-    python_requires='>=3.4',
+    python_requires='>=3.5.3',
     install_requires=[
         'm_lib.full>=1.0',
     ],
@@ -40,6 +39,6 @@ setup(
         'html': ['beautifulsoup4', 'lxml'],
         'requests': ['requests[socks]'],
         'curl': ['pycurl', 'certifi'],
-        'aiohttp': ['aiohttp', 'aioftp'],
+        'aiohttp': ['aiohttp>=2.3.2', 'aiohttp-socks', 'aioftp'],
     },
 )