Feat: For the robot based on requests allow to use a proxy

diff --git a/Makefile b/Makefile
index 6dc83ed792c64ae3aa74790e3d5fd5035d8c4bf7..b49ebfb4d7b001a766c1f775dc218cdcf5415efb 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -6,7 +6,7 @@
 # __license__ = "GNU GPL"
 #
 
-VERSION=5.1.0
+VERSION=5.2.0
 
 EXAMPLE_SHELL=\
    bkmk-add bkmk-add.py bkmk-chk bkmk-publish bkmk-rsync bkmk-sort bkmk2db \

diff --git a/Robots/bkmk_rrequests.py b/Robots/bkmk_rrequests.py
index d1bf98bfca2e9b5c758cbf762d51697922f614c6..24b09d6803ef763b24406267599f61bb2df73911 100644 (file)
--- a/Robots/bkmk_rrequests.py
+++ b/Robots/bkmk_rrequests.py
@@ -20,6 +20,10 @@ requests_ftp.monkeypatch_session()
 
 
 class robot_requests(robot_base):
+    # Pass proxy from the environment like this:
+    # BKMK_ROBOT=requests:proxy=socks5h%3a//localhost%3a1080
+    proxy = None
+
     def get(self, bookmark, url, accept_charset=False):
         try:
             r = requests.Session().get(
@@ -27,11 +31,21 @@ class robot_requests(robot_base):
         except requests.RequestException as e:
             error = str(e)
             self.log('   Error: %s' % error)
-            return error, None, None, None, None
-        else:
-            if r.is_redirect:
-                return None, r.status_code, r.next.url, None, None
-            return None, None, None, r.headers, r.content
+            if self.proxy:
+                error = None
+                self.log('   Retrying with the proxy...')
+                try:
+                    r = requests.get(
+                        url, timeout=self.timeout, allow_redirects=False,
+                        proxies={'http': self.proxy, 'https': self.proxy})
+                except requests.RequestException as e:
+                    error = str(e)
+                    self.log('   Proxy error: %s' % error)
+            if error is not None:
+                return error, None, None, None, None
+        if r.is_redirect:
+            return None, r.status_code, r.next.url, None, None
+        return None, None, None, r.headers, r.content
 
     def get_ftp_welcome(self):
         return ''  # Alas, requests_ftp doesn't store welcome message

diff --git a/bkmk-venv b/bkmk-venv
index ad9257ba1c38441665974e98f645bad05ad31045..f36a2d5c65288d052fb78fcec3adba72886931d1 100644 (file)
--- a/bkmk-venv
+++ b/bkmk-venv
@@ -6,6 +6,6 @@ if [ -z "$VIRTUAL_ENV" ]; then
          { python3 -m virtualenv .venv || python3 -m venv .venv; } &&
          . .venv/bin/activate &&
          pip install --compile --upgrade beautifulsoup4 lxml m_lib.full \
-         requests requests-ftp
+         "requests[socks]" requests-ftp
     } || exit 1
 fi

diff --git a/bkmk_objects.py b/bkmk_objects.py
index 1f27c988ec8b7956942064ddbb1885aebd64dfb3..bd5c71377bda4e71d3ee7322567a4a68516ff2b5 100644 (file)
--- a/bkmk_objects.py
+++ b/bkmk_objects.py
@@ -14,7 +14,7 @@ __all__ = ['Folder', 'Bookmark', 'Ruler', 'Walker', 'Writer', 'Robot',
            ]
 
 
-from urllib.parse import urlsplit, quote
+from urllib.parse import urlsplit, quote, unquote
 import os
 
 BKMK_FORMAT = os.environ.get("BKMK_FORMAT", "MOZILLA")
@@ -236,7 +236,7 @@ def parse_params(param_str):
     param_list = {}
     for param in params:
         key, value = param.split('=', 1)
-        param_list[key] = value
+        param_list[key] = unquote(value)
     return main_param, param_list
 
 

diff --git a/doc/ANNOUNCE b/doc/ANNOUNCE
index b6775e9efd4445d0ccd974bc1c1fae1fee854178..2eaac9547df7001476c8ce800fa7279dec39abaa 100644 (file)
--- a/doc/ANNOUNCE
+++ b/doc/ANNOUNCE
@@ -6,6 +6,10 @@ WHAT IS IT
 bookmarks.html.
 
 WHAT'S NEW
+Version 5.2.0 (2024-03-02)
+
+   For the robot based on requests allow to use a proxy.
+
 Version 5.1.0 (2024-03-01)
 
    Robot based on requests.

diff --git a/doc/ChangeLog b/doc/ChangeLog
index 9267bae315c8e5369541567184cd3424c45e83dd..204a30671346633668427ca3fb97c21e84878cbd 100644 (file)
--- a/doc/ChangeLog
+++ b/doc/ChangeLog
@@ -1,3 +1,7 @@
+Version 5.2.0 (2024-03-02)
+
+   For the robot based on requests allow to use a proxy.
+
 Version 5.1.0 (2024-03-01)
 
    Robot based on requests.