From 7d468d4d0d0002f8fb6b5ac61a43331d5b442b57 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Thu, 15 Aug 2024 23:05:29 +0300 Subject: [PATCH] Refactor(bkmk_robot_base): --- Robots/bkmk_robot_base.py | 51 ++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/Robots/bkmk_robot_base.py b/Robots/bkmk_robot_base.py index 7c2e08d..a8a15d5 100644 --- a/Robots/bkmk_robot_base.py +++ b/Robots/bkmk_robot_base.py @@ -8,7 +8,7 @@ __author__ = "Oleg Broytman " __copyright__ = "Copyright (C) 2000-2024 PhiloSoft Design" __license__ = "GNU GPL" -__all__ = ['robot_base', 'get_error'] +__all__ = ['robot_base'] from base64 import b64encode @@ -25,10 +25,6 @@ from parse_html import parse_html # Fake headers to pretend this is a real browser -_user_agent = 'Mozilla/5.0 (X11; Linux x86_64; rv:120.0) ' \ - 'Gecko/20100101 Firefox/120.0' -_x_user_agent = 'bookmarks_db (Python %d.%d.%d)' % sys.version_info[:3] - request_headers = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,' 'image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8', @@ -36,12 +32,13 @@ request_headers = { 'Cache-Control': 'max-age=300', 'Connection': 'close', 'Referer': '/', - 'User-Agent': _user_agent, - 'X-User-Agent': _x_user_agent, + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:120.0) ' + 'Gecko/20100101 Firefox/120.0', + 'X-User-Agent': 'bookmarks_db (Python %d.%d.%d)' % sys.version_info[:3], } -reloc_dict = { +_reloc_dict = { 301: "perm1.", 302: "temp2.", 303: "temp3.", @@ -51,7 +48,7 @@ reloc_dict = { } -def get_error(e): +def _get_error(e): if isinstance(e, str): return e @@ -62,12 +59,11 @@ def get_error(e): return "(%s)" % ' '.join(s) -# Icon cache; maps URL to a tuple (content type, data) -# or None if there is no icon. -icons = {} - - class robot_base(Robot): + # Icon cache; maps URL to a tuple (content type, data) + # or None if there is no icon. + icons = {} + # Pass proxy from the environment like this: # BKMK_ROBOT=curl,requests:proxy=http%3a//localhost%3a8080 # BKMK_ROBOT=requests,curl:proxy=socks5h%3a//localhost%3a1080 @@ -90,9 +86,8 @@ class robot_base(Robot): Robot.__init__(self, *args, **kw) socket.setdefaulttimeout(self.timeout) - global _x_user_agent - _x_user_agent = '%s %s' % (_x_user_agent, self.version_str()) - request_headers['X-User-Agent'] = _x_user_agent + request_headers['X-User-Agent'] = '%s %s' % ( + request_headers['X-User-Agent'], self.version_str()) def version_str(self): return self.__class__.__name__ @@ -196,10 +191,11 @@ class robot_base(Robot): if icon_url_log.startswith('data:'): icon_url_log = icon_url_log[:50] + '...' self.log(" looking for icon at: %s" % icon_url_log) - if icon_url in icons: - if icons[icon_url]: + if icon_url in self.icons: + if self.icons[icon_url]: bookmark.icon_href = icon_url - content_type, bookmark.icon = icons[icon_url] + content_type, bookmark.icon = \ + self.icons[icon_url] self.log(" cached icon : %s" % content_type) else: @@ -209,7 +205,7 @@ class robot_base(Robot): icon_url[len('data:'):].split(',', 1) bookmark.icon_href = bookmark.icon = icon_url self.log(" got data icon : %s" % content_type) - icons[icon_url] = (content_type, icon_url) + self.icons[icon_url] = (content_type, icon_url) else: try: _icon_url = icon_url @@ -237,7 +233,7 @@ class robot_base(Robot): self.log(" no icon : %s %s" % (etype, emsg)) etype = emsg = _ = None - icons[icon_url] = None + self.icons[icon_url] = None else: content_type = icon_headers["Content-Type"] if content_type and ( @@ -261,15 +257,14 @@ class robot_base(Robot): bookmark.icon = "data:%s;base64,%s" \ % (content_type, b64encode(icon_data).decode('ascii')) - icons[icon_url] = (content_type, - bookmark.icon - ) + self.icons[icon_url] = ( + content_type, bookmark.icon) else: self.log(" no icon : " "bad content type '%s'" % content_type ) - icons[icon_url] = None + self.icons[icon_url] = None if parser and parser.refresh: refresh = parser.refresh try: @@ -325,7 +320,7 @@ class robot_base(Robot): return except socket.error as e: - bookmark.error = get_error(e) + bookmark.error = _get_error(e) self.log(bookmark.error) except: @@ -392,7 +387,7 @@ class robot_base(Robot): return None, None, None, headers, content def set_redirect(self, bookmark, errcode, newurl): - bookmark.moved = moved = "(%s) to %s" % (reloc_dict[errcode], newurl) + bookmark.moved = moved = "(%s) to %s" % (_reloc_dict[errcode], newurl) try: moved.encode('ascii') except UnicodeEncodeError: -- 2.39.5