1 #! /usr/bin/env python3
2 """Robot interface - check URLs from bookmarks database
4 This file is a part of Bookmarks database and Internet robot.
7 from __future__ import print_function
11 __author__ = "Oleg Broytman <phd@phdru.name>"
12 __copyright__ = "Copyright (C) 2000-2024 PhiloSoft Design"
13 __license__ = "GNU GPL"
17 from getopt import getopt
18 optlist, args = getopt(sys.argv[1:], "ise")
24 for _opt, _arg in optlist:
37 print("Broytman check_urls_db, "
38 "Copyright (C) 1997-2024 PhiloSoft Design")
41 sys.stderr.write("check_urls_db: too many arguments\n")
42 sys.stderr.write("Usage: check_urls_db [-ise]\n")
46 show_pbar = sys.stderr.isatty()
50 from m_lib.pbar.tty_pbar import ttyProgressBar
54 from m_lib.flog import makelog, openlog
56 log = openlog("check.log")
57 log("chk_urls restarted for errors")
59 print("chk_urls restarted for errors")
61 log = makelog("check.log")
62 log("check_urls_db started")
64 print(" check_urls_db: normal start")
65 log.outfile.reconfigure(encoding='utf-8')
67 from storage import storage
70 from robots import robot
74 sys.stdout.write("Loading %s: " % storage.filename)
77 root_folder = storage.load()
78 from bkmk_objects import make_linear, break_tree
79 make_linear(root_folder)
80 objects = len(root_folder.linear)
87 s = "Rechecking errors: "
94 pbar = ttyProgressBar(0, objects)
103 for object_no in range(objects):
105 pbar.display(object_no+1)
107 object = root_folder.linear[object_no]
108 object_count = object_count + 1
110 if object.isBookmark:
112 if (href.startswith('place:') # Firefox SmartBookmarks
113 or '%s' in href): # Bookmark with keyword
114 log("Skipped %s" % href)
118 if hasattr(object, "error"):
119 delattr(object, "error")
124 log("Already checked %s" % href)
125 old_object = root_folder.linear[checked[href]]
127 "last_visit", "last_modified",
128 "error", "no_error", "moved", "size", "md5", "real_title",
129 "last_tested", "test_time", "icon", "charset",
131 if hasattr(old_object, attr_name):
132 setattr(object, attr_name,
133 getattr(old_object, attr_name))
135 log("Checking %s" % href)
136 rcode = robot.check_url(object)
139 checked[href] = object_no
140 urls_no = urls_no + 1
142 size = size + int(object.size)
143 except (AttributeError, TypeError, ValueError):
144 pass # Some object does not have a size :(
146 log("Interrupted by user (^C)")
155 print(object_count, "objects passed")
156 print(urls_no, "URLs checked")
157 print(size, "bytes eaten")
159 break_tree(root_folder.linear)
160 storage.store(root_folder)
163 log("check_urls_db finished ok")
167 if __name__ == '__main__':