Robots/bkmk_robot_base.py

   1 """Base class for robots
   2
   3 This file is a part of Bookmarks database and Internet robot.
   4
   5 """
   6
   7 __author__ = "Oleg Broytman <phd@phdru.name>"
   8 __copyright__ = "Copyright (C) 2000-2024 PhiloSoft Design"
   9 __license__ = "GNU GPL"
  10
  11 __all__ = ['robot_base', 'get_error']
  12
  13
  14 from base64 import b64encode
  15 from urllib.parse import urlsplit, urljoin
  16 import sys
  17 import socket
  18 import time
  19
  20 from m_lib.md5wrapper import md5wrapper
  21 from m_lib.net.www.util import parse_time
  22
  23 from bkmk_objects import Robot
  24 from parse_html import parse_html
  25
  26
  27 # Fake headers to pretend this is a real browser
  28 _user_agent = "Mozilla/5.0 (X11; U; Linux 2.6 i686; en)"
  29 " Gecko/20001221 Firefox/2.0.0"
  30 _x_user_agent = "bookmarks_db (Python %d.%d.%d)" % sys.version_info[:3]
  31
  32 request_headers = {
  33     'Accept': '*/*',
  34     'Accept-Language': 'ru,en',
  35     'Cache-Control': 'max-age=300',
  36     'Connection': 'close',
  37     'Referer': '/',
  38     'User-Agent': _user_agent,
  39     'X-User-Agent': _x_user_agent,
  40 }
  41
  42
  43 reloc_dict = {
  44   301: "perm1.",
  45   302: "temp2.",
  46   303: "temp3.",
  47   307: "temp7.",
  48   308: "temp8.",
  49   "html": "html"
  50 }
  51
  52
  53 def get_error(e):
  54     if isinstance(e, str):
  55         return e
  56
  57     else:
  58         s = []
  59         for i in e:
  60             s.append("'%s'" % str(i).replace('\n', "\\n"))
  61         return "(%s)" % ' '.join(s)
  62
  63
  64 # Icon cache; maps URL to a tuple (content type, data)
  65 # or None if there is no icon.
  66 icons = {}
  67
  68
  69 class robot_base(Robot):
  70     timeout = 60
  71
  72     def __init__(self, *args, **kw):
  73         Robot.__init__(self, *args, **kw)
  74         socket.setdefaulttimeout(int(self.timeout))
  75
  76     def check_url(self, bookmark):
  77         try:
  78             self.start = int(time.time())
  79             bookmark.icon = None
  80
  81             split_results = urlsplit(bookmark.href)
  82             url_type, netloc, url_path, query, url_tag = split_results
  83             url_host = split_results.hostname
  84
  85             url = "%s://%s%s" % (url_type, url_host, url_path)
  86             error, redirect_code, redirect_to, headers, content = \
  87                 self.get(bookmark, url, True)
  88
  89             if error:
  90                 bookmark.error = error
  91                 return 1
  92
  93             if redirect_code:
  94                 self.set_redirect(bookmark, redirect_code, redirect_to)
  95                 return 1
  96
  97             size = 0
  98             last_modified = None
  99
 100             if headers:
 101                 try:
 102                     size = headers["Content-Length"]
 103                 except KeyError:
 104                     pass
 105
 106                 try:
 107                     last_modified = headers["Last-Modified"]
 108                 except KeyError:
 109                     pass
 110
 111                 if last_modified:
 112                     last_modified = parse_time(last_modified)
 113
 114             if not size:  # Could be None from headers
 115                 size = len(content)
 116
 117             if last_modified:
 118                 last_modified = str(int(last_modified))
 119             else:
 120                 last_modified = bookmark.last_visit
 121
 122             bookmark.size = size
 123             bookmark.last_modified = last_modified
 124
 125             charset = None
 126             if headers:
 127                 try:
 128                     content_type = headers["Content-Type"]
 129                     self.log("   Content-Type   : %s" % content_type)
 130                     if content_type is None:
 131                         if b'html' in content.lower():
 132                             content_type = 'text/html'
 133                         else:
 134                             content_type = 'text/plain'
 135                         self.log("   Set Content-Type to: %s"
 136                                  % content_type)
 137                     try:
 138                         # extract charset from
 139                         # "text/html; charset=UTF-8, foo; bar"
 140                         content_type, charset = content_type.split(';', 1)
 141                         content_type = content_type.strip()
 142                         charset = charset.split('=')[1].strip().split(',')[0]
 143                         self.log("   HTTP charset   : %s" % charset)
 144                     except (ValueError, IndexError):
 145                         charset = None
 146                         self.log("   no charset in Content-Type header")
 147                     is_html = False
 148                     for ctype in ("text/html", "application/xhtml+xml"):
 149                         if content_type.startswith(ctype):
 150                             is_html = True
 151                             break
 152                     content_stripped = content.strip()
 153                     if content_stripped and charset:
 154                         try:
 155                             content_stripped = content_stripped.decode(
 156                                 charset, 'replace')
 157                         except LookupError:
 158                             charset = None
 159                             self.log("   unknown charset "
 160                                      "in Content-Type header")
 161                     if content_stripped and is_html:
 162                         parser = parse_html(
 163                             content_stripped, charset, self.log)
 164                         if charset:
 165                             bookmark.charset = charset
 166                         elif parser and parser.meta_charset:
 167                             bookmark.charset = parser.meta_charset
 168                         if parser:
 169                             bookmark.real_title = parser.title
 170                             icon = parser.icon
 171                         else:
 172                             icon = None
 173                         if not icon:
 174                             icon = "/favicon.ico"
 175                         icon_url = urljoin(
 176                             "%s://%s%s" % (url_type, url_host, url_path), icon)
 177                         self.log("   looking for icon at: %s" % icon_url)
 178                         if icon_url in icons:
 179                             if icons[icon_url]:
 180                                 bookmark.icon_href = icon_url
 181                                 content_type, bookmark.icon = icons[icon_url]
 182                                 self.log("   cached icon: %s" % content_type)
 183                             else:
 184                                 self.log("   cached icon: no icon")
 185                         elif icon_url.startswith('data:'):
 186                             content_type, icon_data = \
 187                                 icon_url[len('data:'):].split(',', 1)
 188                             bookmark.icon_href = bookmark.icon = icon_url
 189                             self.log("   got data icon  : %s" % content_type)
 190                             icons[icon_url] = (content_type, icon_url)
 191                         else:
 192                             try:
 193                                 _icon_url = icon_url
 194                                 for i in range(8):
 195                                     error, icon_redirect_code, \
 196                                         icon_redirect_to, icon_headers, \
 197                                         icon_data = \
 198                                         self.get(bookmark, _icon_url)
 199                                     if icon_redirect_code:
 200                                         _icon_url = icon_redirect_to
 201                                         self.log("   redirect to : %s"
 202                                                  % _icon_url)
 203                                     else:
 204                                         if icon_data is None:
 205                                             raise IOError("No icon")
 206                                         break
 207                                 else:
 208                                     raise IOError("Too many redirects")
 209                             except:
 210                                 etype, emsg, _ = sys.exc_info()
 211                                 self.log("   no icon        : %s %s"
 212                                          % (etype, emsg))
 213                                 etype = emsg = _ = None
 214                                 icons[icon_url] = None
 215                             else:
 216                                 content_type = icon_headers["Content-Type"]
 217                                 if content_type and (
 218                                     content_type.startswith("application/")
 219                                     or content_type.startswith("image/")
 220                                     or content_type.startswith("text/plain")
 221                                 ):
 222                                     bookmark.icon_href = icon_url
 223                                     self.log("   got icon       : %s"
 224                                              % content_type)
 225                                     if (
 226                                         content_type.startswith("application/")
 227                                         or content_type.startswith(
 228                                             "text/plain")
 229                                     ):
 230                                         self.log("   non-image content type,"
 231                                                  " assume x-icon")
 232                                         content_type = 'image/x-icon'
 233                                     if not isinstance(icon_data, bytes):
 234                                         icon_data = icon_data.encode('latin1')
 235                                     bookmark.icon = "data:%s;base64,%s" \
 236                                         % (content_type, b64encode(icon_data))
 237                                     icons[icon_url] = (content_type,
 238                                                        bookmark.icon
 239                                                        )
 240                                 else:
 241                                     self.log("   no icon        : "
 242                                              "bad content type '%s'"
 243                                              % content_type
 244                                              )
 245                                     icons[icon_url] = None
 246                         if parser and parser.refresh:
 247                             refresh = parser.refresh
 248                             try:
 249                                 url = refresh.split('=', 1)[1]
 250                             except IndexError:
 251                                 url = "self"
 252                             try:
 253                                 timeout = float(refresh.split(';')[0])
 254                             except (IndexError, ValueError):
 255                                 self.set_redirect(bookmark, "html",
 256                                                   "Bad redirect to %s (%s)"
 257                                                   % (url, refresh)
 258                                                   )
 259                             else:
 260                                 try:
 261                                     timeout = int(refresh.split(';')[0])
 262                                 except ValueError:
 263                                     pass  # float timeout
 264                                 self.set_redirect(bookmark, "html",
 265                                                   "%s (%s sec)"
 266                                                   % (url, timeout)
 267                                                   )
 268                     elif charset:
 269                         bookmark.charset = charset
 270
 271                     if not content_stripped:
 272                         self.log("   empty response, no content")
 273                     if not is_html:
 274                         self.log("   not html")
 275                 except KeyError as key:
 276                     self.log("   no header: %s" % key)
 277
 278             md5 = md5wrapper()
 279             if url_type == "ftp":  # Pass welcome message through MD5
 280                 ftp_welcome = self.get_ftp_welcome()
 281                 if not isinstance(ftp_welcome, bytes):
 282                     ftp_welcome = ftp_welcome.encode(charset or 'utf-8')
 283                 md5.update(ftp_welcome)
 284
 285             if isinstance(content, bytes):
 286                 md5.update(content)
 287             else:
 288                 md5.update(content.encode(charset or 'utf-8'))
 289             bookmark.md5 = str(md5)
 290
 291         except EOFError:
 292             bookmark.error = "Unexpected EOF (FTP server closed connection)"
 293             self.log('   EOF: %s' % bookmark.error)
 294
 295         except KeyboardInterrupt:
 296             self.log("Keyboard interrupt (^C)")
 297             return 0
 298
 299         except socket.error as e:
 300             bookmark.error = get_error(e)
 301             self.log(bookmark.error)
 302
 303         except:
 304             import traceback
 305             traceback.print_exc()
 306             bookmark.error = "Exception!"
 307             self.log('   Exception: %s' % bookmark.error)
 308
 309         finally:
 310             self.finish_check_url(bookmark)
 311
 312         # Tested
 313         return 1
 314
 315     def set_redirect(self, bookmark, errcode, newurl):
 316         bookmark.moved = moved = "(%s) to %s" % (reloc_dict[errcode], newurl)
 317         try:
 318             moved.encode('ascii')
 319         except UnicodeEncodeError:
 320             try:
 321                 moved = moved.encode(bookmark.charset)
 322             except (LookupError, TypeError, UnicodeEncodeError):
 323                 moved = moved.encode('utf-8')
 324         self.log('   Moved: %s' % moved)
 325
 326     def finish_check_url(self, bookmark):
 327         start = self.start
 328         bookmark.last_tested = str(start)
 329         now = int(time.time())
 330         bookmark.test_time = str(now - start)