From: Oleg Broytman <phd@phdru.name>
Date: Sun, 3 Mar 2024 10:10:13 +0000 (+0300)
Subject: Feat(Robots/bkmk_rrequests): Add request headers
X-Git-Tag: 5.2.2~2
X-Git-Url: https://git.phdru.name/?a=commitdiff_plain;h=2ec71bc7b14b39fc84806fd614631dc6f2b98b36;p=bookmarks_db.git

Feat(Robots/bkmk_rrequests): Add request headers
---

diff --git a/Robots/bkmk_rrequests.py b/Robots/bkmk_rrequests.py
index 114d670..54b4c62 100644
--- a/Robots/bkmk_rrequests.py
+++ b/Robots/bkmk_rrequests.py
@@ -16,7 +16,7 @@ from urllib.parse import urlsplit
 import requests
 import requests_ftp
 
-from Robots.bkmk_robot_base import robot_base
+from Robots.bkmk_robot_base import robot_base, request_headers
 
 requests_ftp.monkeypatch_session()
 
@@ -38,16 +38,23 @@ class robot_requests(robot_base):
         if url_host in self.proxy_error:
             return 'proxy error', None, None, None, None
 
+        if accept_charset and bookmark.charset:
+            headers = request_headers.copy()
+            headers['Accept-Charset'] = bookmark.charset
+        else:
+            headers = request_headers
+
         if url_host in self.proxy_ok:
             self.log('   Immediately trying with the proxy')
-            error, r = request_get(url, self.timeout, self.proxy)
+            error, r = request_get(url, headers, self.timeout, self.proxy)
         else:
-            error, r = request_get(url, self.timeout, None)
+            error, r = request_get(url, headers, self.timeout, None)
             if error is not None:
                 self.log('   Error: %s' % error)
                 if self.proxy:
                     self.log('   Retrying with the proxy...')
-                    error, r = request_get(url, self.timeout, self.proxy)
+                    error, r = request_get(url, headers,
+                                           self.timeout, self.proxy)
                     if error is None:
                         self.proxy_ok.add(url_host)
         if error is not None:
@@ -64,7 +71,7 @@ class robot_requests(robot_base):
         return ''  # Alas, requests_ftp doesn't store welcome message
 
 
-def request_get(url, timeout, proxy):
+def request_get(url, headers, timeout, proxy):
     if proxy:
         proxies = {'http': proxy, 'https': proxy}
     else:
@@ -72,7 +79,8 @@ def request_get(url, timeout, proxy):
 
     try:
         r = requests.Session().get(
-            url, timeout=timeout, allow_redirects=False, proxies=proxies)
+            url, headers=headers, timeout=timeout,
+            allow_redirects=False, proxies=proxies)
     except requests.RequestException as e:
         return str(e), None
     else:
diff --git a/doc/ChangeLog b/doc/ChangeLog
index fa40ada..4096a6f 100644
--- a/doc/ChangeLog
+++ b/doc/ChangeLog
@@ -1,3 +1,7 @@
+Version 5.2.2 (2024-03-03)
+
+   Robots/bkmk_rrequests: Add request headers.
+
 Version 5.2.1 (2024-03-02)
 
    Speedup second access through proxy.