From 011586eca56d3bde3fef2087579e6cbc0682f5b4 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Sat, 2 Mar 2024 00:02:57 +0300 Subject: [PATCH] Refactor(Rebobt/requests) --- Robots/bkmk_rrequests.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/Robots/bkmk_rrequests.py b/Robots/bkmk_rrequests.py index 24b09d6..e1f0019 100644 --- a/Robots/bkmk_rrequests.py +++ b/Robots/bkmk_rrequests.py @@ -25,27 +25,34 @@ class robot_requests(robot_base): proxy = None def get(self, bookmark, url, accept_charset=False): - try: - r = requests.Session().get( - url, timeout=self.timeout, allow_redirects=False) - except requests.RequestException as e: - error = str(e) + error, r = request_get(url, self.timeout, None) + if error is not None: self.log(' Error: %s' % error) if self.proxy: - error = None self.log(' Retrying with the proxy...') - try: - r = requests.get( - url, timeout=self.timeout, allow_redirects=False, - proxies={'http': self.proxy, 'https': self.proxy}) - except requests.RequestException as e: - error = str(e) - self.log(' Proxy error: %s' % error) - if error is not None: - return error, None, None, None, None + error, r = request_get(url, self.timeout, self.proxy) + if error is not None: + if self.proxy: + self.log(' Proxy error: %s' % error) + return error, None, None, None, None if r.is_redirect: return None, r.status_code, r.next.url, None, None return None, None, None, r.headers, r.content def get_ftp_welcome(self): return '' # Alas, requests_ftp doesn't store welcome message + + +def request_get(url, timeout, proxy): + if proxy: + proxies = {'http': proxy, 'https': proxy} + else: + proxies = None + + try: + r = requests.Session().get( + url, timeout=timeout, allow_redirects=False, proxies=proxies) + except requests.RequestException as e: + return str(e), None + else: + return None, r -- 2.39.2