3 Robot interface - check URLs from bookmarks database
5 Written by Broytman. Copyright (C) 2000-2010 PhiloSoft Design.
10 tempfname = "check_urls.tmp"
14 from getopt import getopt
15 optlist, args = getopt(sys.argv[1:], "ise")
21 for _opt, _arg in optlist:
34 print "Broytman check_urls, Copyright (C) 1997-2010 PhiloSoft Design"
37 sys.stderr.write("check_urls: too many arguments\n")
38 sys.stderr.write("Usage: check_urls [-ise]\n")
42 show_pbar = sys.stderr.isatty()
46 from m_lib.pbar.tty_pbar import ttyProgressBar
50 from m_lib.flog import makelog, openlog
52 log = openlog("check.log")
53 log("chk_urls restarted for errors")
55 print "chk_urls restarted for errors"
57 log = makelog("check.log")
58 log("check_urls started")
60 print " check_urls: normal start"
62 from storage import storage
65 from robots import robot
66 robot = robot(tempfname, log)
69 sys.stdout.write("Loading %s: " % storage.filename)
72 root_folder = storage.load()
73 from bkmk_objects import make_linear, break_tree
74 make_linear(root_folder)
75 objects = len(root_folder.linear)
82 s = "Rechecking errors: "
89 pbar = ttyProgressBar(0, objects)
98 for object_no in range(objects):
100 pbar.display(object_no+1)
102 object = root_folder.linear[object_no]
103 object_count = object_count + 1
105 if object.isBookmark:
106 if object.href.startswith('place:'): # Firefox SmartBookmarks
107 log("Skipped %s" % object.href)
111 if hasattr(object, "error"):
112 delattr(object, "error")
116 if checked.has_key(object.href):
117 log("Already checked %s" % object.href)
118 old_object = root_folder.linear[checked[object.href]]
119 for attr_name in ("last_visit", "last_modified",
120 "error", "no_error", "moved", "size", "md5", "real_title",
121 "last_tested", "test_time", "icon", "charset"):
122 if hasattr(old_object, attr_name):
123 setattr(object, attr_name, getattr(old_object, attr_name))
125 log("Checking %s" % object.href)
126 rcode = robot.check_url(object)
129 checked[object.href] = object_no
130 urls_no = urls_no + 1
132 size = size + int(object.size)
133 except (AttributeError, TypeError, ValueError):
134 pass # Some object does not have a size :(
136 log("Interrupted by user (^C)")
145 print object_count, "objects passed"
146 print urls_no, "URLs checked"
147 print size, "bytes eaten"
149 break_tree(root_folder.linear)
150 storage.store(root_folder)
153 log("check_urls finished ok")
162 if __name__ == '__main__':