Robots/bkmk_rforking.py

   1 """Forking robot
   2
   3 This file is a part of Bookmarks database and Internet robot.
   4 """
   5
   6 __author__ = "Oleg Broytman <phd@phdru.name>"
   7 __copyright__ = "Copyright (C) 2000-2023 PhiloSoft Design"
   8 __license__ = "GNU GPL"
   9
  10 __all__ = ['robot_forking']
  11
  12
  13 import os
  14 import sys
  15
  16 try:
  17     import cPickle as pickle
  18 except ImportError:
  19     import pickle
  20
  21 from subproc import Subprocess, RecordFile
  22 from bkmk_objects import Robot
  23
  24
  25 # This is to catch 'close failed: [Errno 9] Bad file descriptor' message
  26 # from os.close() in Subprocess.die() and errors from the subprocess.
  27 sys.stderr = open("err.log", 'at')
  28
  29 check_subp = None
  30 subp_pipe = None
  31
  32
  33 def stop_subp(log):
  34     global check_subp, subp_pipe
  35     if check_subp:
  36         if log: log("   restarting hanging subprocess")
  37         del check_subp
  38     del subp_pipe
  39
  40
  41 def restart_subp(log):
  42     global check_subp, subp_pipe
  43     stop_subp(log)
  44
  45     check_subp = Subprocess("%s/Robots/bkmk_rforking_sub.py"
  46                             % os.path.dirname(sys.argv[0]),
  47                             control_stderr=True)
  48     subp_pipe = RecordFile(check_subp)
  49
  50
  51 _set_subproc = True
  52
  53
  54 class robot_forking(Robot):
  55     subproc = 'urllib'  # Default subprocess
  56
  57     def check_url(self, bookmark):
  58         global _set_subproc
  59         if _set_subproc:
  60             _set_subproc = False
  61
  62             subproc = self.subproc
  63             subproc_attrs = []
  64             for attr in dir(self):
  65                 if attr.startswith('subproc_'):
  66                     subproc_attrs.append(
  67                         (attr[len('subproc_'):], getattr(self, attr))
  68                     )
  69             if subproc_attrs:
  70                 subproc += ':' + ':'.join(
  71                     ['='.join((k, v)) for k, v in subproc_attrs]
  72                 )
  73             os.environ['BKMK_ROBOT'] = subproc
  74
  75         if not check_subp:
  76             restart_subp(self.log)  # Not restart, just start afresh
  77
  78         try:
  79             save_parent = bookmark.parent
  80             bookmark.parent = None
  81             subp_pipe.write_record(pickle.dumps(bookmark))
  82
  83             if check_subp.waitForPendingChar(60):  # wait a minute
  84                 new_b = pickle.loads(subp_pipe.read_record())
  85                 for attr in (
  86                     "error", "no_error",
  87                     "moved", "size", "md5", "real_title",
  88                     "last_tested", "last_modified", "test_time",
  89                     "icon", "icon_href", "charset",
  90                 ):
  91                     if hasattr(new_b, attr):
  92                         setattr(bookmark, attr, getattr(new_b, attr))
  93             else:
  94                 bookmark.error = "Subprocess connection timed out"
  95                 restart_subp(self.log)
  96
  97             bookmark.parent = save_parent
  98
  99             while True:
 100                 error = check_subp.readPendingErrLine()
 101                 if not error:
 102                     break
 103                 sys.stderr.write("(subp) %s" % error)
 104             sys.stderr.flush()
 105
 106         except KeyboardInterrupt:
 107             return 0
 108
 109         # Tested
 110         return 1
 111
 112     def stop(self):
 113         stop_subp(None)  # Stop subprocess; do not log restarting