From: Oleg Broytman <phd@phdru.name>
Date: Fri, 1 Mar 2024 20:57:56 +0000 (+0300)
Subject: Feat: For the robot based on requests allow to use a proxy
X-Git-Tag: 5.2.0^0
X-Git-Url: https://git.phdru.name/?a=commitdiff_plain;h=ed7127ea175cab12642244ebd2d86276adcb2e25;p=bookmarks_db.git

Feat: For the robot based on requests allow to use a proxy
---

diff --git a/Makefile b/Makefile
index 6dc83ed..b49ebfb 100644
--- a/Makefile
+++ b/Makefile
@@ -6,7 +6,7 @@
 # __license__ = "GNU GPL"
 #
 
-VERSION=5.1.0
+VERSION=5.2.0
 
 EXAMPLE_SHELL=\
    bkmk-add bkmk-add.py bkmk-chk bkmk-publish bkmk-rsync bkmk-sort bkmk2db \
diff --git a/Robots/bkmk_rrequests.py b/Robots/bkmk_rrequests.py
index d1bf98b..24b09d6 100644
--- a/Robots/bkmk_rrequests.py
+++ b/Robots/bkmk_rrequests.py
@@ -20,6 +20,10 @@ requests_ftp.monkeypatch_session()
 
 
 class robot_requests(robot_base):
+    # Pass proxy from the environment like this:
+    # BKMK_ROBOT=requests:proxy=socks5h%3a//localhost%3a1080
+    proxy = None
+
     def get(self, bookmark, url, accept_charset=False):
         try:
             r = requests.Session().get(
@@ -27,11 +31,21 @@ class robot_requests(robot_base):
         except requests.RequestException as e:
             error = str(e)
             self.log('   Error: %s' % error)
-            return error, None, None, None, None
-        else:
-            if r.is_redirect:
-                return None, r.status_code, r.next.url, None, None
-            return None, None, None, r.headers, r.content
+            if self.proxy:
+                error = None
+                self.log('   Retrying with the proxy...')
+                try:
+                    r = requests.get(
+                        url, timeout=self.timeout, allow_redirects=False,
+                        proxies={'http': self.proxy, 'https': self.proxy})
+                except requests.RequestException as e:
+                    error = str(e)
+                    self.log('   Proxy error: %s' % error)
+            if error is not None:
+                return error, None, None, None, None
+        if r.is_redirect:
+            return None, r.status_code, r.next.url, None, None
+        return None, None, None, r.headers, r.content
 
     def get_ftp_welcome(self):
         return ''  # Alas, requests_ftp doesn't store welcome message
diff --git a/bkmk-venv b/bkmk-venv
index ad9257b..f36a2d5 100644
--- a/bkmk-venv
+++ b/bkmk-venv
@@ -6,6 +6,6 @@ if [ -z "$VIRTUAL_ENV" ]; then
          { python3 -m virtualenv .venv || python3 -m venv .venv; } &&
          . .venv/bin/activate &&
          pip install --compile --upgrade beautifulsoup4 lxml m_lib.full \
-         requests requests-ftp
+         "requests[socks]" requests-ftp
     } || exit 1
 fi
diff --git a/bkmk_objects.py b/bkmk_objects.py
index 1f27c98..bd5c713 100644
--- a/bkmk_objects.py
+++ b/bkmk_objects.py
@@ -14,7 +14,7 @@ __all__ = ['Folder', 'Bookmark', 'Ruler', 'Walker', 'Writer', 'Robot',
            ]
 
 
-from urllib.parse import urlsplit, quote
+from urllib.parse import urlsplit, quote, unquote
 import os
 
 BKMK_FORMAT = os.environ.get("BKMK_FORMAT", "MOZILLA")
@@ -236,7 +236,7 @@ def parse_params(param_str):
     param_list = {}
     for param in params:
         key, value = param.split('=', 1)
-        param_list[key] = value
+        param_list[key] = unquote(value)
     return main_param, param_list
 
 
diff --git a/doc/ANNOUNCE b/doc/ANNOUNCE
index b6775e9..2eaac95 100644
--- a/doc/ANNOUNCE
+++ b/doc/ANNOUNCE
@@ -6,6 +6,10 @@ WHAT IS IT
 bookmarks.html.
 
 WHAT'S NEW
+Version 5.2.0 (2024-03-02)
+
+   For the robot based on requests allow to use a proxy.
+
 Version 5.1.0 (2024-03-01)
 
    Robot based on requests.
diff --git a/doc/ChangeLog b/doc/ChangeLog
index 9267bae..204a306 100644
--- a/doc/ChangeLog
+++ b/doc/ChangeLog
@@ -1,3 +1,7 @@
+Version 5.2.0 (2024-03-02)
+
+   For the robot based on requests allow to use a proxy.
+
 Version 5.1.0 (2024-03-01)
 
    Robot based on requests.