From f84406ac3fd5b69919ff801e1db82a56a7186a10 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Wed, 15 Nov 2023 02:27:46 +0300 Subject: [PATCH] Feat(robots): Handle HTTP redirect 308 --- Robots/bkmk_robot_base.py | 3 ++- Robots/bkmk_rurllib.py | 2 ++ Robots/bkmk_rurllib2.py | 2 +- Robots/bkmk_rurllib_py3.py | 2 ++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Robots/bkmk_robot_base.py b/Robots/bkmk_robot_base.py index 85c704a..1d762d9 100644 --- a/Robots/bkmk_robot_base.py +++ b/Robots/bkmk_robot_base.py @@ -29,10 +29,11 @@ from parse_html import parse_html reloc_dict = { - 301: "perm.", + 301: "perm1.", 302: "temp2.", 303: "temp3.", 307: "temp7.", + 308: "temp8.", "html": "html" } diff --git a/Robots/bkmk_rurllib.py b/Robots/bkmk_rurllib.py index 93dbae8..067799c 100644 --- a/Robots/bkmk_rurllib.py +++ b/Robots/bkmk_rurllib.py @@ -41,6 +41,8 @@ class MyURLopener(urllib.URLopener): http_error_303 = http_error_301 # Error 307 -- relocated (temporarily) http_error_307 = http_error_301 + # Error 308 -- relocated (permanently) + http_error_308 = http_error_301 # Error 401 -- authentication required def http_error_401(self, url, fp, errcode, errmsg, headers, data=None): diff --git a/Robots/bkmk_rurllib2.py b/Robots/bkmk_rurllib2.py index 66f0460..1233c70 100644 --- a/Robots/bkmk_rurllib2.py +++ b/Robots/bkmk_rurllib2.py @@ -66,7 +66,7 @@ class robot_urllib2(robot_base): response = urllib2.urlopen(request) except urllib2.HTTPError as e: - if e.code in (301, 302, 303, 307): + if e.code in (301, 302, 303, 307, 308): return None, e.code, e.hdrs['Location'], None, None else: self.log(' HTTP Error %s: %s' % (e.code, e.msg)) diff --git a/Robots/bkmk_rurllib_py3.py b/Robots/bkmk_rurllib_py3.py index 9493a58..1daec38 100644 --- a/Robots/bkmk_rurllib_py3.py +++ b/Robots/bkmk_rurllib_py3.py @@ -43,6 +43,8 @@ class MyURLopener(urllib.request.URLopener): http_error_303 = http_error_301 # Error 307 -- relocated (temporarily) http_error_307 = http_error_301 + # Error 308 -- relocated (permanently) + http_error_308 = http_error_301 # Error 401 -- authentication required def http_error_401(self, url, fp, errcode, errmsg, headers, data=None): -- 2.39.5