Change default subprocess robot to urllib2

[bookmarks_db.git] / Robots / bkmk_rforking.py

diff --git a/Robots/bkmk_rforking.py b/Robots/bkmk_rforking.py
index 1fb18a0adb4a884a8a8d2b841df6baabcbdc7075..4ad998403417398b15643f3de2809c4c7c382f22 100644 (file)
--- a/Robots/bkmk_rforking.py
+++ b/Robots/bkmk_rforking.py
@@ -1,20 +1,30 @@
-"""
-   Forking robot
+"""Forking robot
 
-   Written by BroytMann. Copyright (C) 2000-2007 PhiloSoft Design
+This file is a part of Bookmarks database and Internet robot.
 """
 
+__author__ = "Oleg Broytman <phd@phdru.name>"
+__copyright__ = "Copyright (C) 2000-2012 PhiloSoft Design"
+__license__ = "GNU GPL"
+
+__all__ = ['robot_forking']
+
+
+import sys, os
 
 try:
-   import cPickle
-   pickle = cPickle
+   import cPickle as pickle
 except ImportError:
    import pickle
 
-import sys, os
 from subproc import Subprocess, RecordFile
+from bkmk_objects import Robot
 
 
+# This is to catch 'close failed: [Errno 9] Bad file descriptor' message
+# from os.close() in Subprocess.die() and errors from the subprocess.
+sys.stderr = open("err.log", 'a')
+
 check_subp = None
 subp_pipe = None
 
@@ -29,29 +39,44 @@ def restart_subp(log):
    global check_subp, subp_pipe
    stop_subp(log)
 
-   check_subp = Subprocess("%s/Robots/bkmk_rforking_sub.py" % os.path.dirname(sys.argv[0]))
+   check_subp = Subprocess("%s/Robots/bkmk_rforking_sub.py" % os.path.dirname(sys.argv[0]),
+      control_stderr=True)
    subp_pipe = RecordFile(check_subp)
 
 
-from bkmk_objects import Robot
+_set_subproc = True
 
 class robot_forking(Robot):
-   def check_url(self, bookmark, url_type, url_rest):
+   subproc = 'urllib2' # Default subprocess
+
+   def check_url(self, bookmark):
+      global _set_subproc
+      if _set_subproc:
+         _set_subproc = False
+
+         subproc = self.subproc
+         subproc_attrs = []
+         for attr in dir(self):
+             if attr.startswith('subproc_'):
+                 subproc_attrs.append((attr[len('subproc_'):], getattr(self, attr)))
+         if subproc_attrs:
+             subproc += ':' + ':'.join(['='.join((k,v)) for k,v in subproc_attrs])
+         os.environ['BKMK_ROBOT'] = subproc
+
       if not check_subp:
          restart_subp(self.log) # Not restart, just start afresh
 
       try:
          save_parent = bookmark.parent
          bookmark.parent = None
-
-         bookmark.tempfname = self.tempfname
          subp_pipe.write_record(pickle.dumps(bookmark))
 
-         if check_subp.waitForPendingChar(900): # wait 15 minutes
+         if check_subp.waitForPendingChar(60): # wait a minute
             new_b = pickle.loads(subp_pipe.read_record())
             for attr in ("error", "no_error",
                   "moved", "size", "md5", "real_title",
-                  "last_tested", "last_modified", "test_time", "icon"):
+                  "last_tested", "last_modified", "test_time",
+                  "icon", "icon_href"):
                if hasattr(new_b, attr):
                   setattr(bookmark, attr, getattr(new_b, attr))
          else:
@@ -60,12 +85,18 @@ class robot_forking(Robot):
 
          bookmark.parent = save_parent
 
+         while True:
+            error = check_subp.readPendingErrLine()
+            if not error:
+               break
+            sys.stderr.write("(subp) " + error)
+         sys.stderr.flush()
+
       except KeyboardInterrupt:
          return 0
 
       # Tested
       return 1
 
-
    def stop(self):
       stop_subp(None) # Stop subprocess; do not log restarting