X-Git-Url: https://git.phdru.name/?a=blobdiff_plain;f=Robots%2Fbkmk_robot_base.py;h=d6a87d31cd41682ee3d48549868442814a15cebe;hb=cb9c36b39ed72cd1fa272130d2bcf162a89c3013;hp=0600e7338d2b71c41492d73858acf72d0818ae84;hpb=c88cb7a75e7caf1d67466cfa107981d95115fa0c;p=bookmarks_db.git diff --git a/Robots/bkmk_robot_base.py b/Robots/bkmk_robot_base.py index 0600e73..d6a87d3 100644 --- a/Robots/bkmk_robot_base.py +++ b/Robots/bkmk_robot_base.py @@ -5,7 +5,7 @@ This file is a part of Bookmarks database and Internet robot. """ __author__ = "Oleg Broytman " -__copyright__ = "Copyright (C) 2000-2017 PhiloSoft Design" +__copyright__ = "Copyright (C) 2000-2023 PhiloSoft Design" __license__ = "GNU GPL" __all__ = ['robot_base', 'get_error'] @@ -45,8 +45,10 @@ def get_error(e): return "(%s)" % ' '.join(s) -icons = {} # Icon cache; maps URL to a tuple (content type, data) - # or None if there is no icon. +# Icon cache; maps URL to a tuple (content type, data) +# or None if there is no icon. +icons = {} + class robot_base(Robot): timeout = 60 @@ -62,10 +64,12 @@ class robot_base(Robot): url_type, url_rest = urllib.splittype(bookmark.href) url_host, url_path = urllib.splithost(url_rest) - url_path, url_tag = urllib.splittag(url_path) + url_path, url_tag = urllib.splittag(url_path) # noqa: E221 + # multiple spaces before operator url = "%s://%s%s" % (url_type, url_host, url_path) - error, redirect_code, redirect_to, headers, content = self.get(bookmark, url, True) + error, redirect_code, redirect_to, headers, content = \ + self.get(bookmark, url, True) if error: bookmark.error = error @@ -103,7 +107,7 @@ class robot_base(Robot): bookmark.last_modified = last_modified md5 = md5wrapper() - if url_type == "ftp": # Pass welcome message through MD5 + if url_type == "ftp": # Pass welcome message through MD5 md5.update(self.get_ftp_welcome()) md5.update(content) @@ -114,7 +118,8 @@ class robot_base(Robot): content_type = headers["Content-Type"] self.log(" Content-Type: %s" % content_type) try: - # extract charset from "text/html; foo; charset=UTF-8, bar; baz;" + # extract charset from + # "text/html; foo; charset=UTF-8, bar; baz;" content_type, charset = content_type.split(';', 1) content_type = content_type.strip() charset = charset.split('=')[1].strip().split(',')[0] @@ -137,7 +142,8 @@ class robot_base(Robot): icon = None if not icon: icon = "/favicon.ico" - icon_url = urljoin("%s://%s%s" % (url_type, url_host, url_path), icon) + icon_url = urljoin( + "%s://%s%s" % (url_type, url_host, url_path), icon) self.log(" looking for icon at: %s" % icon_url) if icon_url in icons: if icons[icon_url]: @@ -150,12 +156,14 @@ class robot_base(Robot): try: _icon_url = icon_url for i in range(8): - error, icon_redirect_code, icon_redirect_to, \ - icon_headers, icon_data = \ - self.get(bookmark, _icon_url) + error, icon_redirect_code, \ + icon_redirect_to, icon_headers, \ + icon_data = \ + self.get(bookmark, _icon_url) if icon_redirect_code: _icon_url = icon_redirect_to - self.log(" redirect to : %s" % _icon_url) + self.log(" redirect to : %s" + % _icon_url) else: if icon_data is None: raise IOError("No icon") @@ -164,24 +172,36 @@ class robot_base(Robot): raise IOError("Too many redirects") except: etype, emsg, tb = sys.exc_info() - self.log(" no icon : %s %s" % (etype, emsg)) + self.log(" no icon : %s %s" + % (etype, emsg)) etype = emsg = tb = None icons[icon_url] = None else: content_type = icon_headers["Content-Type"] if content_type.startswith("application/") \ - or content_type.startswith("image/") \ - or content_type.startswith("text/plain"): + or content_type.startswith("image/") \ + or content_type.startswith("text/plain"): bookmark.icon_href = icon_url - self.log(" got icon : %s" % content_type) - if content_type.startswith("application/") \ - or content_type.startswith("text/plain"): - self.log(" non-image content type, assume x-icon") + self.log(" got icon : %s" + % content_type) + if ( + content_type.startswith("application/") + or content_type.startswith( + "text/plain") + ): + self.log(" non-image content type," + " assume x-icon") content_type = 'image/x-icon' - bookmark.icon = "data:%s;base64,%s" % (content_type, b64encode(icon_data)) - icons[icon_url] = (content_type, bookmark.icon) + bookmark.icon = "data:%s;base64,%s" \ + % (content_type, b64encode(icon_data)) + icons[icon_url] = (content_type, + bookmark.icon + ) else: - self.log(" no icon : bad content type '%s'" % content_type) + self.log(" no icon :" + "bad content type '%s'" + % content_type + ) icons[icon_url] = None if parser and parser.refresh: refresh = parser.refresh @@ -192,13 +212,19 @@ class robot_base(Robot): try: timeout = float(refresh.split(';')[0]) except (IndexError, ValueError): - self.set_redirect(bookmark, "html", "Bad redirect to %s (%s)" % (url, refresh)) + self.set_redirect(bookmark, "html", + "Bad redirect to %s (%s)" + % (url, refresh) + ) else: try: timeout = int(refresh.split(';')[0]) except ValueError: - pass # float timeout - self.set_redirect(bookmark, "html", "%s (%s sec)" % (url, timeout)) + pass # float timeout + self.set_redirect(bookmark, "html", + "%s (%s sec)" + % (url, timeout) + ) except KeyError as key: self.log(" no header: %s" % key)