1 """Robot based on requests
3 This file is a part of Bookmarks database and Internet robot.
7 __author__ = "Oleg Broytman <phd@phdru.name>"
8 __copyright__ = "Copyright (C) 2024 PhiloSoft Design"
9 __license__ = "GNU GPL"
11 __all__ = ['robot_requests']
17 from Robots.bkmk_robot_base import robot_base
19 requests_ftp.monkeypatch_session()
22 class robot_requests(robot_base):
23 # Pass proxy from the environment like this:
24 # BKMK_ROBOT=requests:proxy=socks5h%3a//localhost%3a1080
27 def get(self, bookmark, url, accept_charset=False):
28 error, r = request_get(url, self.timeout, None)
30 self.log(' Error: %s' % error)
32 self.log(' Retrying with the proxy...')
33 error, r = request_get(url, self.timeout, self.proxy)
36 self.log(' Proxy error: %s' % error)
37 return error, None, None, None, None
39 return None, r.status_code, r.next.url, None, None
40 return None, None, None, r.headers, r.content
42 def get_ftp_welcome(self):
43 return '' # Alas, requests_ftp doesn't store welcome message
46 def request_get(url, timeout, proxy):
48 proxies = {'http': proxy, 'https': proxy}
53 r = requests.Session().get(
54 url, timeout=timeout, allow_redirects=False, proxies=proxies)
55 except requests.RequestException as e: