2 """Robot interface - check URLs from bookmarks database
4 This file is a part of Bookmarks database and Internet robot.
7 __author__ = "Oleg Broytman <phd@phdru.name>"
8 __copyright__ = "Copyright (C) 2000-2012 PhiloSoft Design"
9 __license__ = "GNU GPL"
15 from getopt import getopt
16 optlist, args = getopt(sys.argv[1:], "ise")
22 for _opt, _arg in optlist:
35 print "Broytman check_urls, Copyright (C) 1997-2010 PhiloSoft Design"
38 sys.stderr.write("check_urls: too many arguments\n")
39 sys.stderr.write("Usage: check_urls [-ise]\n")
43 show_pbar = sys.stderr.isatty()
47 from m_lib.pbar.tty_pbar import ttyProgressBar
51 from m_lib.flog import makelog, openlog
53 log = openlog("check.log")
54 log("chk_urls restarted for errors")
56 print "chk_urls restarted for errors"
58 log = makelog("check.log")
59 log("check_urls started")
61 print " check_urls: normal start"
63 from storage import storage
66 from robots import robot
70 sys.stdout.write("Loading %s: " % storage.filename)
73 root_folder = storage.load()
74 from bkmk_objects import make_linear, break_tree
75 make_linear(root_folder)
76 objects = len(root_folder.linear)
83 s = "Rechecking errors: "
90 pbar = ttyProgressBar(0, objects)
99 for object_no in range(objects):
101 pbar.display(object_no+1)
103 object = root_folder.linear[object_no]
104 object_count = object_count + 1
106 if object.isBookmark:
108 if (href.startswith('place:') # Firefox SmartBookmarks
109 or '%s' in href): # Bookmark with keyword
110 log("Skipped %s" % href)
114 if hasattr(object, "error"):
115 delattr(object, "error")
119 if checked.has_key(href):
120 log("Already checked %s" % href)
121 old_object = root_folder.linear[checked[href]]
122 for attr_name in ("last_visit", "last_modified",
123 "error", "no_error", "moved", "size", "md5", "real_title",
124 "last_tested", "test_time", "icon", "charset"):
125 if hasattr(old_object, attr_name):
126 setattr(object, attr_name, getattr(old_object, attr_name))
128 log("Checking %s" % href)
129 rcode = robot.check_url(object)
132 checked[href] = object_no
133 urls_no = urls_no + 1
135 size = size + int(object.size)
136 except (AttributeError, TypeError, ValueError):
137 pass # Some object does not have a size :(
139 log("Interrupted by user (^C)")
148 print object_count, "objects passed"
149 print urls_no, "URLs checked"
150 print size, "bytes eaten"
152 break_tree(root_folder.linear)
153 storage.store(root_folder)
156 log("check_urls finished ok")
160 if __name__ == '__main__':