Robots/bkmk_robot_base.py

   1 """Base class for robots
   2
   3 This file is a part of Bookmarks database and Internet robot.
   4
   5 """
   6
   7 __author__ = "Oleg Broytman <phd@phdru.name>"
   8 __copyright__ = "Copyright (C) 2000-2024 PhiloSoft Design"
   9 __license__ = "GNU GPL"
  10
  11 __all__ = ['robot_base', 'get_error']
  12
  13
  14 from base64 import b64encode
  15 from urllib.parse import urlsplit, urljoin
  16 import sys
  17 import socket
  18 import time
  19
  20 from m_lib.md5wrapper import md5wrapper
  21 from m_lib.net.www.util import parse_time
  22
  23 from bkmk_objects import Robot
  24 from parse_html import parse_html
  25
  26
  27 # Fake headers to pretend this is a real browser
  28 _user_agent = "Mozilla/5.0 (X11; U; Linux 2.6 i686; en)"
  29 " Gecko/20001221 Firefox/2.0.0"
  30 _x_user_agent = "bookmarks_db (Python %d.%d.%d)" % sys.version_info[:3]
  31
  32 request_headers = {
  33     'Accept': '*/*',
  34     'Accept-Language': 'ru,en',
  35     'Cache-Control': 'max-age=300',
  36     'Connection': 'close',
  37     'Referer': '/',
  38     'User-Agent': _user_agent,
  39     'X-User-Agent': _x_user_agent,
  40 }
  41
  42
  43 reloc_dict = {
  44   301: "perm1.",
  45   302: "temp2.",
  46   303: "temp3.",
  47   307: "temp7.",
  48   308: "temp8.",
  49   "html": "html"
  50 }
  51
  52
  53 def get_error(e):
  54     if isinstance(e, str):
  55         return e
  56
  57     else:
  58         s = []
  59         for i in e:
  60             s.append("'%s'" % str(i).replace('\n', "\\n"))
  61         return "(%s)" % ' '.join(s)
  62
  63
  64 # Icon cache; maps URL to a tuple (content type, data)
  65 # or None if there is no icon.
  66 icons = {}
  67
  68
  69 class robot_base(Robot):
  70     timeout = 60
  71
  72     def __init__(self, *args, **kw):
  73         Robot.__init__(self, *args, **kw)
  74         socket.setdefaulttimeout(int(self.timeout))
  75
  76     def check_url(self, bookmark):
  77         try:
  78             self.start = int(time.time())
  79             bookmark.icon = None
  80
  81             split_results = urlsplit(bookmark.href)
  82             url_type, netloc, url_path, query, url_tag = split_results
  83             url_host = split_results.hostname
  84
  85             if query:
  86                 url = "%s://%s%s?%s" % (url_type, url_host, url_path, query)
  87             else:
  88                 url = "%s://%s%s" % (url_type, url_host, url_path)
  89             error, redirect_code, redirect_to, headers, content = \
  90                 self.get(bookmark, url, True)
  91
  92             if error:
  93                 bookmark.error = error
  94                 return 1
  95
  96             if redirect_code:
  97                 self.set_redirect(bookmark, redirect_code, redirect_to)
  98                 return 1
  99
 100             size = 0
 101             last_modified = None
 102
 103             if headers:
 104                 try:
 105                     size = headers["Content-Length"]
 106                 except KeyError:
 107                     pass
 108
 109                 try:
 110                     last_modified = headers["Last-Modified"]
 111                 except KeyError:
 112                     pass
 113
 114                 if last_modified:
 115                     last_modified = parse_time(last_modified)
 116
 117             if not size:  # Could be None from headers
 118                 size = len(content)
 119
 120             if last_modified:
 121                 last_modified = str(int(last_modified))
 122             else:
 123                 last_modified = bookmark.last_visit
 124
 125             bookmark.size = size
 126             bookmark.last_modified = last_modified
 127
 128             charset = None
 129             if headers:
 130                 try:
 131                     content_type = headers["Content-Type"]
 132                     self.log("   Content-Type   : %s" % content_type)
 133                     if content_type is None:
 134                         if b'html' in content.lower():
 135                             content_type = 'text/html'
 136                         else:
 137                             content_type = 'text/plain'
 138                         self.log("   Set Content-Type to: %s"
 139                                  % content_type)
 140                     try:
 141                         # extract charset from
 142                         # "text/html; charset=UTF-8, foo; bar"
 143                         content_type, charset = content_type.split(';', 1)
 144                         content_type = content_type.strip()
 145                         charset = charset.split('=')[1].strip().split(',')[0]
 146                         self.log("   HTTP charset   : %s" % charset)
 147                     except (ValueError, IndexError):
 148                         charset = None
 149                         self.log("   no charset in Content-Type header")
 150                     is_html = False
 151                     for ctype in ("text/html", "application/xhtml+xml"):
 152                         if content_type.startswith(ctype):
 153                             is_html = True
 154                             break
 155                     content_stripped = content.strip()
 156                     if content_stripped and charset:
 157                         try:
 158                             content_stripped = content_stripped.decode(
 159                                 charset, 'replace')
 160                         except LookupError:
 161                             charset = None
 162                             self.log("   unknown charset "
 163                                      "in Content-Type header")
 164                     if content_stripped and is_html:
 165                         parser = parse_html(
 166                             content_stripped, charset, self.log)
 167                         if charset:
 168                             bookmark.charset = charset
 169                         elif parser and parser.meta_charset:
 170                             bookmark.charset = parser.meta_charset
 171                         if parser:
 172                             bookmark.real_title = parser.title
 173                             icon = parser.icon
 174                         else:
 175                             icon = None
 176                         if not icon:
 177                             icon = "/favicon.ico"
 178                         icon_url = urljoin(
 179                             "%s://%s%s" % (url_type, url_host, url_path), icon)
 180                         self.log("   looking for icon at: %s" % icon_url)
 181                         if icon_url in icons:
 182                             if icons[icon_url]:
 183                                 bookmark.icon_href = icon_url
 184                                 content_type, bookmark.icon = icons[icon_url]
 185                                 self.log("   cached icon    : %s"
 186                                          % content_type)
 187                             else:
 188                                 self.log("   cached icon    : no icon")
 189                         elif icon_url.startswith('data:'):
 190                             content_type, icon_data = \
 191                                 icon_url[len('data:'):].split(',', 1)
 192                             bookmark.icon_href = bookmark.icon = icon_url
 193                             self.log("   got data icon  : %s" % content_type)
 194                             icons[icon_url] = (content_type, icon_url)
 195                         else:
 196                             try:
 197                                 _icon_url = icon_url
 198                                 for i in range(8):
 199                                     error, icon_redirect_code, \
 200                                         icon_redirect_to, icon_headers, \
 201                                         icon_data = \
 202                                         self.get(bookmark, _icon_url)
 203                                     if icon_redirect_code:
 204                                         _icon_url = icon_redirect_to
 205                                         self.log("   redirect to : %s"
 206                                                  % _icon_url)
 207                                     else:
 208                                         if icon_data is None:
 209                                             raise IOError("No icon")
 210                                         break
 211                                 else:
 212                                     raise IOError("Too many redirects")
 213                             except:
 214                                 etype, emsg, _ = sys.exc_info()
 215                                 self.log("   no icon        : %s %s"
 216                                          % (etype, emsg))
 217                                 etype = emsg = _ = None
 218                                 icons[icon_url] = None
 219                             else:
 220                                 content_type = icon_headers["Content-Type"]
 221                                 if content_type and (
 222                                     content_type.startswith("application/")
 223                                     or content_type.startswith("image/")
 224                                     or content_type.startswith("text/plain")
 225                                 ):
 226                                     bookmark.icon_href = icon_url
 227                                     self.log("   got icon       : %s"
 228                                              % content_type)
 229                                     if (
 230                                         content_type.startswith("application/")
 231                                         or content_type.startswith(
 232                                             "text/plain")
 233                                     ):
 234                                         self.log("   non-image content type,"
 235                                                  " assume x-icon")
 236                                         content_type = 'image/x-icon'
 237                                     if not isinstance(icon_data, bytes):
 238                                         icon_data = icon_data.encode('latin1')
 239                                     bookmark.icon = "data:%s;base64,%s" \
 240                                         % (content_type, b64encode(icon_data))
 241                                     icons[icon_url] = (content_type,
 242                                                        bookmark.icon
 243                                                        )
 244                                 else:
 245                                     self.log("   no icon        : "
 246                                              "bad content type '%s'"
 247                                              % content_type
 248                                              )
 249                                     icons[icon_url] = None
 250                         if parser and parser.refresh:
 251                             refresh = parser.refresh
 252                             try:
 253                                 url = refresh.split('=', 1)[1]
 254                             except IndexError:
 255                                 url = "self"
 256                             try:
 257                                 timeout = float(refresh.split(';')[0])
 258                             except (IndexError, ValueError):
 259                                 self.set_redirect(bookmark, "html",
 260                                                   "Bad redirect to %s (%s)"
 261                                                   % (url, refresh)
 262                                                   )
 263                             else:
 264                                 try:
 265                                     timeout = int(refresh.split(';')[0])
 266                                 except ValueError:
 267                                     pass  # float timeout
 268                                 self.set_redirect(bookmark, "html",
 269                                                   "%s (%s sec)"
 270                                                   % (url, timeout)
 271                                                   )
 272                     elif charset:
 273                         bookmark.charset = charset
 274
 275                     if not content_stripped:
 276                         self.log("   empty response, no content")
 277                     if not is_html:
 278                         self.log("   not html")
 279                 except KeyError as key:
 280                     self.log("   no header: %s" % key)
 281
 282             md5 = md5wrapper()
 283             if url_type == "ftp":  # Pass welcome message through MD5
 284                 ftp_welcome = self.get_ftp_welcome()
 285                 if not isinstance(ftp_welcome, bytes):
 286                     ftp_welcome = ftp_welcome.encode(charset or 'utf-8')
 287                 md5.update(ftp_welcome)
 288
 289             if isinstance(content, bytes):
 290                 md5.update(content)
 291             else:
 292                 md5.update(content.encode(charset or 'utf-8'))
 293             bookmark.md5 = str(md5)
 294
 295         except EOFError:
 296             bookmark.error = "Unexpected EOF (FTP server closed connection)"
 297             self.log('   EOF: %s' % bookmark.error)
 298
 299         except KeyboardInterrupt:
 300             self.log("Keyboard interrupt (^C)")
 301             return 0
 302
 303         except socket.error as e:
 304             bookmark.error = get_error(e)
 305             self.log(bookmark.error)
 306
 307         except:
 308             import traceback
 309             traceback.print_exc()
 310             bookmark.error = "Exception!"
 311             self.log('   Exception: %s' % bookmark.error)
 312
 313         finally:
 314             self.finish_check_url(bookmark)
 315
 316         # Tested
 317         return 1
 318
 319     def set_redirect(self, bookmark, errcode, newurl):
 320         bookmark.moved = moved = "(%s) to %s" % (reloc_dict[errcode], newurl)
 321         try:
 322             moved.encode('ascii')
 323         except UnicodeEncodeError:
 324             try:
 325                 moved = moved.encode(bookmark.charset)
 326             except (LookupError, TypeError, UnicodeEncodeError):
 327                 moved = moved.encode('utf-8')
 328         self.log('   Moved: %s' % moved)
 329
 330     def finish_check_url(self, bookmark):
 331         start = self.start
 332         bookmark.last_tested = str(start)
 333         now = int(time.time())
 334         bookmark.test_time = str(now - start)