Robots/bkmk_robot_base.py

   1 """Base class for robots
   2
   3 This file is a part of Bookmarks database and Internet robot.
   4
   5 """
   6
   7 __author__ = "Oleg Broytman <phd@phdru.name>"
   8 __copyright__ = "Copyright (C) 2000-2024 PhiloSoft Design"
   9 __license__ = "GNU GPL"
  10
  11 __all__ = ['robot_base', 'get_error']
  12
  13
  14 from base64 import b64encode
  15 from urllib.parse import urljoin
  16 import sys
  17 import socket
  18 import time
  19
  20 from m_lib.md5wrapper import md5wrapper
  21 from m_lib.net.www.util import parse_time
  22
  23 from bkmk_objects import Robot
  24 from parse_html import parse_html
  25
  26
  27 # Fake headers to pretend this is a real browser
  28 _user_agent = "Mozilla/5.0 (X11; U; Linux 2.6 i686; en)"
  29 " Gecko/20001221 Firefox/2.0.0"
  30 _x_user_agent = "bookmarks_db (Python %d.%d.%d)" % sys.version_info[:3]
  31
  32 request_headers = {
  33     'Accept': '*/*',
  34     'Accept-Language': 'ru,en',
  35     'Cache-Control': 'max-age=300',
  36     'Connection': 'close',
  37     'Referer': '/',
  38     'User-Agent': _user_agent,
  39     'X-User-Agent': _x_user_agent,
  40 }
  41
  42
  43 reloc_dict = {
  44   301: "perm1.",
  45   302: "temp2.",
  46   303: "temp3.",
  47   307: "temp7.",
  48   308: "temp8.",
  49   "html": "html"
  50 }
  51
  52
  53 def get_error(e):
  54     if isinstance(e, str):
  55         return e
  56
  57     else:
  58         s = []
  59         for i in e:
  60             s.append("'%s'" % str(i).replace('\n', "\\n"))
  61         return "(%s)" % ' '.join(s)
  62
  63
  64 # Icon cache; maps URL to a tuple (content type, data)
  65 # or None if there is no icon.
  66 icons = {}
  67
  68
  69 class robot_base(Robot):
  70     timeout = 60
  71
  72     def __init__(self, *args, **kw):
  73         Robot.__init__(self, *args, **kw)
  74         socket.setdefaulttimeout(int(self.timeout))
  75
  76     def check_url(self, bookmark):
  77         try:
  78             self.start = int(time.time())
  79             bookmark.icon = None
  80
  81             error, redirect_code, redirect_to, headers, content = \
  82                 self.get(bookmark, bookmark.href, True)
  83
  84             if error:
  85                 bookmark.error = error
  86                 return 1
  87
  88             if redirect_code:
  89                 self.set_redirect(bookmark, redirect_code, redirect_to)
  90                 return 1
  91
  92             size = 0
  93             last_modified = None
  94
  95             if headers:
  96                 try:
  97                     size = headers["Content-Length"]
  98                 except KeyError:
  99                     pass
 100
 101                 try:
 102                     last_modified = headers["Last-Modified"]
 103                 except KeyError:
 104                     pass
 105
 106                 if last_modified:
 107                     last_modified = parse_time(last_modified)
 108
 109             if not size:  # Could be None from headers
 110                 size = len(content)
 111
 112             if last_modified:
 113                 last_modified = str(int(last_modified))
 114             else:
 115                 last_modified = bookmark.last_visit
 116
 117             bookmark.size = size
 118             bookmark.last_modified = last_modified
 119
 120             charset = None
 121             if headers:
 122                 try:
 123                     content_type = headers["Content-Type"]
 124                     self.log("   Content-Type   : %s" % content_type)
 125                     if content_type is None:
 126                         if b'html' in content.lower():
 127                             content_type = 'text/html'
 128                         else:
 129                             content_type = 'text/plain'
 130                         self.log("   Set Content-Type to: %s"
 131                                  % content_type)
 132                     try:
 133                         # extract charset from
 134                         # "text/html; charset=UTF-8, foo; bar"
 135                         content_type, charset = content_type.split(';', 1)
 136                         content_type = content_type.strip()
 137                         charset = charset.split('=')[1].strip().split(',')[0]
 138                         self.log("   HTTP charset   : %s" % charset)
 139                     except (ValueError, IndexError):
 140                         charset = None
 141                         self.log("   no charset in Content-Type header")
 142                     is_html = False
 143                     for ctype in ("text/html", "application/xhtml+xml"):
 144                         if content_type.startswith(ctype):
 145                             is_html = True
 146                             break
 147                     content_stripped = content.strip()
 148                     if content_stripped and charset:
 149                         try:
 150                             content_stripped = content_stripped.decode(
 151                                 charset, 'replace')
 152                         except LookupError:
 153                             charset = None
 154                             self.log("   unknown charset "
 155                                      "in Content-Type header")
 156                     if content_stripped and is_html:
 157                         parser = parse_html(
 158                             content_stripped, charset, self.log)
 159                         if charset:
 160                             bookmark.charset = charset
 161                         elif parser and parser.meta_charset:
 162                             bookmark.charset = parser.meta_charset
 163                         if parser:
 164                             bookmark.real_title = parser.title
 165                             icon = parser.icon
 166                         else:
 167                             icon = None
 168                         if not icon:
 169                             icon = "/favicon.ico"
 170                         icon_url = urljoin(bookmark.href, icon)
 171                         self.log("   looking for icon at: %s" % icon_url)
 172                         if icon_url in icons:
 173                             if icons[icon_url]:
 174                                 bookmark.icon_href = icon_url
 175                                 content_type, bookmark.icon = icons[icon_url]
 176                                 self.log("   cached icon    : %s"
 177                                          % content_type)
 178                             else:
 179                                 self.log("   cached icon    : no icon")
 180                         elif icon_url.startswith('data:'):
 181                             content_type, icon_data = \
 182                                 icon_url[len('data:'):].split(',', 1)
 183                             bookmark.icon_href = bookmark.icon = icon_url
 184                             self.log("   got data icon  : %s" % content_type)
 185                             icons[icon_url] = (content_type, icon_url)
 186                         else:
 187                             try:
 188                                 _icon_url = icon_url
 189                                 for i in range(8):
 190                                     error, icon_redirect_code, \
 191                                         icon_redirect_to, icon_headers, \
 192                                         icon_data = \
 193                                         self.get(bookmark, _icon_url)
 194                                     if icon_redirect_code:
 195                                         _icon_url = icon_redirect_to
 196                                         self.log("   redirect to : %s"
 197                                                  % _icon_url)
 198                                     else:
 199                                         if icon_data is None:
 200                                             raise IOError("No icon")
 201                                         break
 202                                 else:
 203                                     raise IOError("Too many redirects")
 204                             except:
 205                                 etype, emsg, _ = sys.exc_info()
 206                                 self.log("   no icon        : %s %s"
 207                                          % (etype, emsg))
 208                                 etype = emsg = _ = None
 209                                 icons[icon_url] = None
 210                             else:
 211                                 content_type = icon_headers["Content-Type"]
 212                                 if content_type and (
 213                                     content_type.startswith("application/")
 214                                     or content_type.startswith("image/")
 215                                     or content_type.startswith("text/plain")
 216                                 ):
 217                                     bookmark.icon_href = icon_url
 218                                     self.log("   got icon       : %s"
 219                                              % content_type)
 220                                     if (
 221                                         content_type.startswith("application/")
 222                                         or content_type.startswith(
 223                                             "text/plain")
 224                                     ):
 225                                         self.log("   non-image content type,"
 226                                                  " assume x-icon")
 227                                         content_type = 'image/x-icon'
 228                                     if not isinstance(icon_data, bytes):
 229                                         icon_data = icon_data.encode('latin1')
 230                                     bookmark.icon = "data:%s;base64,%s" \
 231                                         % (content_type, b64encode(icon_data))
 232                                     icons[icon_url] = (content_type,
 233                                                        bookmark.icon
 234                                                        )
 235                                 else:
 236                                     self.log("   no icon        : "
 237                                              "bad content type '%s'"
 238                                              % content_type
 239                                              )
 240                                     icons[icon_url] = None
 241                         if parser and parser.refresh:
 242                             refresh = parser.refresh
 243                             try:
 244                                 url = refresh.split('=', 1)[1]
 245                             except IndexError:
 246                                 url = "self"
 247                             try:
 248                                 timeout = float(refresh.split(';')[0])
 249                             except (IndexError, ValueError):
 250                                 self.set_redirect(bookmark, "html",
 251                                                   "Bad redirect to %s (%s)"
 252                                                   % (url, refresh)
 253                                                   )
 254                             else:
 255                                 try:
 256                                     timeout = int(refresh.split(';')[0])
 257                                 except ValueError:
 258                                     pass  # float timeout
 259                                 self.set_redirect(bookmark, "html",
 260                                                   "%s (%s sec)"
 261                                                   % (url, timeout)
 262                                                   )
 263                     elif charset:
 264                         bookmark.charset = charset
 265
 266                     if not content_stripped:
 267                         self.log("   empty response, no content")
 268                     if not is_html:
 269                         self.log("   not html")
 270                 except KeyError as key:
 271                     self.log("   no header: %s" % key)
 272
 273             md5 = md5wrapper()
 274             if bookmark.href.startswith("ftp://"):
 275                 # Pass welcome message through MD5
 276                 ftp_welcome = self.get_ftp_welcome()
 277                 if not isinstance(ftp_welcome, bytes):
 278                     ftp_welcome = ftp_welcome.encode(charset or 'utf-8')
 279                 md5.update(ftp_welcome)
 280
 281             if isinstance(content, bytes):
 282                 md5.update(content)
 283             else:
 284                 md5.update(content.encode(charset or 'utf-8'))
 285             bookmark.md5 = str(md5)
 286
 287         except EOFError:
 288             bookmark.error = "Unexpected EOF (FTP server closed connection)"
 289             self.log('   EOF: %s' % bookmark.error)
 290
 291         except KeyboardInterrupt:
 292             self.log("Keyboard interrupt (^C)")
 293             return 0
 294
 295         except socket.error as e:
 296             bookmark.error = get_error(e)
 297             self.log(bookmark.error)
 298
 299         except:
 300             import traceback
 301             traceback.print_exc()
 302             bookmark.error = "Exception!"
 303             self.log('   Exception: %s' % bookmark.error)
 304
 305         finally:
 306             self.finish_check_url(bookmark)
 307
 308         # Tested
 309         return 1
 310
 311     def set_redirect(self, bookmark, errcode, newurl):
 312         bookmark.moved = moved = "(%s) to %s" % (reloc_dict[errcode], newurl)
 313         try:
 314             moved.encode('ascii')
 315         except UnicodeEncodeError:
 316             try:
 317                 moved = moved.encode(bookmark.charset)
 318             except (LookupError, TypeError, UnicodeEncodeError):
 319                 moved = moved.encode('utf-8')
 320         self.log('   Moved: %s' % moved)
 321
 322     def finish_check_url(self, bookmark):
 323         start = self.start
 324         bookmark.last_tested = str(start)
 325         now = int(time.time())
 326         bookmark.test_time = str(now - start)