X-Git-Url: https://git.phdru.name/?a=blobdiff_plain;ds=sidebyside;f=Robots%2Fbkmk_robot_base.py;h=a03d5c1b16ec430ab17ac8f899655ae9cf9808bb;hb=27d04395e10919fe08fb52b85ceaaa00543596de;hp=fd7237ae0184033797ef30a6a773fe9b200c10de;hpb=3035a4957e210d7a6befcb224f237021b992d285;p=bookmarks_db.git diff --git a/Robots/bkmk_robot_base.py b/Robots/bkmk_robot_base.py index fd7237a..a03d5c1 100644 --- a/Robots/bkmk_robot_base.py +++ b/Robots/bkmk_robot_base.py @@ -5,21 +5,17 @@ This file is a part of Bookmarks database and Internet robot. """ __author__ = "Oleg Broytman " -__copyright__ = "Copyright (C) 2000-2023 PhiloSoft Design" +__copyright__ = "Copyright (C) 2000-2024 PhiloSoft Design" __license__ = "GNU GPL" __all__ = ['robot_base', 'get_error'] from base64 import b64encode +from urllib.parse import urljoin import sys import socket import time -try: - from urllib.parse import splittype, splithost, splittag, urljoin -except ImportError: - from urllib import splittype, splithost, splittag - from urlparse import urljoin from m_lib.md5wrapper import md5wrapper from m_lib.net.www.util import parse_time @@ -28,6 +24,22 @@ from bkmk_objects import Robot from parse_html import parse_html +# Fake headers to pretend this is a real browser +_user_agent = "Mozilla/5.0 (X11; U; Linux 2.6 i686; en)" +" Gecko/20001221 Firefox/2.0.0" +_x_user_agent = "bookmarks_db (Python %d.%d.%d)" % sys.version_info[:3] + +request_headers = { + 'Accept': '*/*', + 'Accept-Language': 'ru,en', + 'Cache-Control': 'max-age=300', + 'Connection': 'close', + 'Referer': '/', + 'User-Agent': _user_agent, + 'X-User-Agent': _x_user_agent, +} + + reloc_dict = { 301: "perm1.", 302: "temp2.", @@ -66,14 +78,8 @@ class robot_base(Robot): self.start = int(time.time()) bookmark.icon = None - url_type, url_rest = splittype(bookmark.href) - url_host, url_path = splithost(url_rest) - url_path, url_tag = splittag(url_path) # noqa: E221 - # multiple spaces before operator - - url = "%s://%s%s" % (url_type, url_host, url_path) error, redirect_code, redirect_to, headers, content = \ - self.get(bookmark, url, True) + self.get(bookmark, bookmark.href, True) if error: bookmark.error = error @@ -111,25 +117,13 @@ class robot_base(Robot): bookmark.size = size bookmark.last_modified = last_modified - md5 = md5wrapper() - if url_type == "ftp": # Pass welcome message through MD5 - ftp_welcome = self.get_ftp_welcome() - if not isinstance(ftp_welcome, bytes): - ftp_welcome = ftp_welcome.encode('utf-8') - md5.update(ftp_welcome) - - if isinstance(content, bytes): - md5.update(content) - else: - md5.update(content.encode('utf-8')) - bookmark.md5 = str(md5) - + charset = None if headers: try: content_type = headers["Content-Type"] self.log(" Content-Type : %s" % content_type) if content_type is None: - if 'html' in content.lower(): + if b'html' in content.lower(): content_type = 'text/html' else: content_type = 'text/plain' @@ -137,7 +131,7 @@ class robot_base(Robot): % content_type) try: # extract charset from - # "text/html; foo; charset=UTF-8, bar; baz;" + # "text/html; charset=UTF-8, foo; bar" content_type, charset = content_type.split(';', 1) content_type = content_type.strip() charset = charset.split('=')[1].strip().split(',')[0] @@ -151,6 +145,14 @@ class robot_base(Robot): is_html = True break content_stripped = content.strip() + if content_stripped and charset: + try: + content_stripped = content_stripped.decode( + charset, 'replace') + except LookupError: + charset = None + self.log(" unknown charset " + "in Content-Type header") if content_stripped and is_html: parser = parse_html( content_stripped, charset, self.log) @@ -165,16 +167,22 @@ class robot_base(Robot): icon = None if not icon: icon = "/favicon.ico" - icon_url = urljoin( - "%s://%s%s" % (url_type, url_host, url_path), icon) + icon_url = urljoin(bookmark.href, icon) self.log(" looking for icon at: %s" % icon_url) if icon_url in icons: if icons[icon_url]: bookmark.icon_href = icon_url content_type, bookmark.icon = icons[icon_url] - self.log(" cached icon: %s" % content_type) + self.log(" cached icon : %s" + % content_type) else: - self.log(" cached icon: no icon") + self.log(" cached icon : no icon") + elif icon_url.startswith('data:'): + content_type, icon_data = \ + icon_url[len('data:'):].split(',', 1) + bookmark.icon_href = bookmark.icon = icon_url + self.log(" got data icon : %s" % content_type) + icons[icon_url] = (content_type, icon_url) else: try: _icon_url = icon_url @@ -218,14 +226,14 @@ class robot_base(Robot): " assume x-icon") content_type = 'image/x-icon' if not isinstance(icon_data, bytes): - icon_data = icon_data.encode('utf-8') + icon_data = icon_data.encode('latin1') bookmark.icon = "data:%s;base64,%s" \ % (content_type, b64encode(icon_data)) icons[icon_url] = (content_type, bookmark.icon ) else: - self.log(" no icon :" + self.log(" no icon : " "bad content type '%s'" % content_type ) @@ -252,6 +260,8 @@ class robot_base(Robot): "%s (%s sec)" % (url, timeout) ) + elif charset: + bookmark.charset = charset if not content_stripped: self.log(" empty response, no content") @@ -260,6 +270,20 @@ class robot_base(Robot): except KeyError as key: self.log(" no header: %s" % key) + md5 = md5wrapper() + if bookmark.href.startswith("ftp://"): + # Pass welcome message through MD5 + ftp_welcome = self.get_ftp_welcome() + if not isinstance(ftp_welcome, bytes): + ftp_welcome = ftp_welcome.encode(charset or 'utf-8') + md5.update(ftp_welcome) + + if isinstance(content, bytes): + md5.update(content) + else: + md5.update(content.encode(charset or 'utf-8')) + bookmark.md5 = str(md5) + except EOFError: bookmark.error = "Unexpected EOF (FTP server closed connection)" self.log(' EOF: %s' % bookmark.error)