From 1b634309d0e17a4c3abc4556cf34923df7354be1 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Wed, 31 Jul 2024 18:14:05 +0300 Subject: [PATCH] Feat(Robots): Return HTTP status code --- Robots/bkmk_robot_base.py | 14 +++++++++----- Robots/bkmk_rrequests.py | 4 ++-- Robots/bkmk_rurllib2.py | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Robots/bkmk_robot_base.py b/Robots/bkmk_robot_base.py index a03d5c1..cc2574a 100644 --- a/Robots/bkmk_robot_base.py +++ b/Robots/bkmk_robot_base.py @@ -78,15 +78,15 @@ class robot_base(Robot): self.start = int(time.time()) bookmark.icon = None - error, redirect_code, redirect_to, headers, content = \ + error, http_status_code, redirect_to, headers, content = \ self.get(bookmark, bookmark.href, True) if error: bookmark.error = error return 1 - if redirect_code: - self.set_redirect(bookmark, redirect_code, redirect_to) + if http_status_code and (http_status_code >= 300): + self.set_redirect(bookmark, http_status_code, redirect_to) return 1 size = 0 @@ -187,11 +187,15 @@ class robot_base(Robot): try: _icon_url = icon_url for i in range(8): - error, icon_redirect_code, \ + error, icon_status_code, \ icon_redirect_to, icon_headers, \ icon_data = \ self.get(bookmark, _icon_url) - if icon_redirect_code: + if error: + raise IOError("No icon") + break + if icon_status_code and ( + icon_status_code >= 300): _icon_url = icon_redirect_to self.log(" redirect to : %s" % _icon_url) diff --git a/Robots/bkmk_rrequests.py b/Robots/bkmk_rrequests.py index 5fe1ca1..317c473 100644 --- a/Robots/bkmk_rrequests.py +++ b/Robots/bkmk_rrequests.py @@ -62,11 +62,11 @@ class robot_requests(robot_base): if error is None: self.proxy_ok.add(url_host) if error is not None: - if self.proxy and error != '404 not_found': + if self.proxy and r.status_code != 404: self.log(' Proxy error : %s' % error) if url_host not in self.proxy_ok: self.proxy_error.add(url_host) - return error, None, None, None, None + return error, r.status_code, None, None, None if r.is_redirect: return None, r.status_code, r.next.url, None, None return None, None, None, r.headers, r.content diff --git a/Robots/bkmk_rurllib2.py b/Robots/bkmk_rurllib2.py index d1b679c..4b8927b 100644 --- a/Robots/bkmk_rurllib2.py +++ b/Robots/bkmk_rurllib2.py @@ -59,7 +59,7 @@ class robot_urllib2(robot_base): else: self.log(' HTTP Error %s: %s' % (e.code, e.msg)) return ("HTTP Error %s: %s" % (e.code, e.msg), - None, None, None, None) + e.code, None, None, None) except urllib2.URLError as e: self.log(' URL Error: %s' % e.reason) -- 2.39.5