-"""
- Forking robot
+"""Forking robot
- Written by BroytMann. Copyright (C) 2000-2007 PhiloSoft Design
+This file is a part of Bookmarks database and Internet robot.
"""
+__author__ = "Oleg Broytman <phd@phdru.name>"
+__copyright__ = "Copyright (C) 2000-2023 PhiloSoft Design"
+__license__ = "GNU GPL"
+
+__all__ = ['robot_forking']
+
+
+import os
+import sys
try:
- import cPickle
- pickle = cPickle
+ import cPickle as pickle
except ImportError:
- import pickle
+ import pickle
-import sys, os
from subproc import Subprocess, RecordFile
+from bkmk_objects import Robot
+# This is to catch 'close failed: [Errno 9] Bad file descriptor' message
+# from os.close() in Subprocess.die() and errors from the subprocess.
+sys.stderr = open("err.log", 'at')
+
check_subp = None
subp_pipe = None
-def stop_subp(log):
- global check_subp, subp_pipe
- if check_subp:
- if log: log(" restarting hanging subprocess")
- del check_subp
- del subp_pipe
-
-def restart_subp(log):
- global check_subp, subp_pipe
- stop_subp(log)
-
- check_subp = Subprocess("%s/Robots/bkmk_rforking_sub.py" % os.path.dirname(sys.argv[0]))
- subp_pipe = RecordFile(check_subp)
-
-from bkmk_objects import Robot
-
-class robot_forking(Robot):
- def check_url(self, bookmark, url_type, url_rest):
- if not check_subp:
- restart_subp(self.log) # Not restart, just start afresh
-
- try:
- save_parent = bookmark.parent
- bookmark.parent = None
+def stop_subp(log):
+ global check_subp, subp_pipe
+ if check_subp:
+ if log: log(" restarting hanging subprocess")
+ del check_subp
+ del subp_pipe
- bookmark.tempfname = self.tempfname
- subp_pipe.write_record(pickle.dumps(bookmark))
- if check_subp.waitForPendingChar(900): # wait 15 minutes
- new_b = pickle.loads(subp_pipe.read_record())
- for attr in ("error", "no_error",
- "moved", "size", "md5", "real_title",
- "last_tested", "last_modified", "test_time", "icon"):
- if hasattr(new_b, attr):
- setattr(bookmark, attr, getattr(new_b, attr))
- else:
- bookmark.error = "Subprocess connection timed out"
- restart_subp(self.log)
+def restart_subp(log):
+ global check_subp, subp_pipe
+ stop_subp(log)
- bookmark.parent = save_parent
+ check_subp = Subprocess("%s/Robots/bkmk_rforking_sub.py"
+ % os.path.dirname(sys.argv[0]),
+ control_stderr=True)
+ subp_pipe = RecordFile(check_subp)
- except KeyboardInterrupt:
- return 0
- # Tested
- return 1
+_set_subproc = True
- def stop(self):
- stop_subp(None) # Stop subprocess; do not log restarting
+class robot_forking(Robot):
+ subproc = 'urllib' # Default subprocess
+
+ def check_url(self, bookmark):
+ global _set_subproc
+ if _set_subproc:
+ _set_subproc = False
+
+ subproc = self.subproc
+ subproc_attrs = []
+ for attr in dir(self):
+ if attr.startswith('subproc_'):
+ subproc_attrs.append(
+ (attr[len('subproc_'):], getattr(self, attr))
+ )
+ if subproc_attrs:
+ subproc += ':' + ':'.join(
+ ['='.join((k, v)) for k, v in subproc_attrs]
+ )
+ os.environ['BKMK_ROBOT'] = subproc
+
+ if not check_subp:
+ restart_subp(self.log) # Not restart, just start afresh
+
+ try:
+ save_parent = bookmark.parent
+ bookmark.parent = None
+ subp_pipe.write_record(pickle.dumps(bookmark))
+
+ if check_subp.waitForPendingChar(60): # wait a minute
+ new_b = pickle.loads(subp_pipe.read_record())
+ for attr in (
+ "error", "no_error",
+ "moved", "size", "md5", "real_title",
+ "last_tested", "last_modified", "test_time",
+ "icon", "icon_href", "charset",
+ ):
+ if hasattr(new_b, attr):
+ setattr(bookmark, attr, getattr(new_b, attr))
+ else:
+ bookmark.error = "Subprocess connection timed out"
+ restart_subp(self.log)
+
+ bookmark.parent = save_parent
+
+ while True:
+ error = check_subp.readPendingErrLine()
+ if not error:
+ break
+ sys.stderr.write("(subp) %s" % error)
+ sys.stderr.flush()
+
+ except KeyboardInterrupt:
+ return 0
+
+ # Tested
+ return 1
+
+ def stop(self):
+ stop_subp(None) # Stop subprocess; do not log restarting