check_urls.py

   1 #! /usr/bin/env python
   2 """
   3    Robot interface - check URLs from bookmarks database
   4
   5    Written by Broytman. Copyright (C) 2000-2010 PhiloSoft Design.
   6 """
   7
   8
   9 import sys, os
  10 tempfname = "check_urls.tmp"
  11
  12
  13 def run():
  14    from getopt import getopt
  15    optlist, args = getopt(sys.argv[1:], "ise")
  16
  17    show_pbar = 1
  18    report_stats = 1
  19    only_errors = 0
  20
  21    for _opt, _arg in optlist:
  22       if _opt == '-i':
  23          show_pbar = 0
  24       if _opt == '-s':
  25          report_stats = 0
  26       if _opt == '-e':
  27          only_errors = 1
  28    try:
  29       del _opt, _arg
  30    except NameError:
  31       pass
  32
  33    if report_stats:
  34       print "Broytman check_urls, Copyright (C) 1997-2010 PhiloSoft Design"
  35
  36    if args:
  37       sys.stderr.write("check_urls: too many arguments\n")
  38       sys.stderr.write("Usage: check_urls [-ise]\n")
  39       sys.exit(1)
  40
  41    if show_pbar:
  42       show_pbar = sys.stderr.isatty()
  43
  44    if show_pbar:
  45       try:
  46          from m_lib.pbar.tty_pbar import ttyProgressBar
  47       except ImportError:
  48          show_pbar = 0
  49
  50    from m_lib.flog import makelog, openlog
  51    if only_errors:
  52       log = openlog("check.log")
  53       log("chk_urls restarted for errors")
  54       if report_stats:
  55          print "chk_urls restarted for errors"
  56    else:
  57       log = makelog("check.log")
  58       log("check_urls started")
  59       if report_stats:
  60          print "   check_urls: normal start"
  61
  62    from storage import storage
  63    storage = storage()
  64
  65    from robots import robot
  66    robot = robot(tempfname, log)
  67
  68    if report_stats:
  69       sys.stdout.write("Loading %s: " % storage.filename)
  70       sys.stdout.flush()
  71
  72    root_folder = storage.load()
  73    from bkmk_objects import make_linear, break_tree
  74    make_linear(root_folder)
  75    objects = len(root_folder.linear)
  76
  77    if report_stats:
  78       print "Ok"
  79
  80    if report_stats:
  81       if only_errors:
  82          s = "Rechecking errors: "
  83       else:
  84          s = "Checking: "
  85       sys.stdout.write(s)
  86       sys.stdout.flush()
  87
  88    if show_pbar:
  89       pbar = ttyProgressBar(0, objects)
  90
  91    urls_no = 0
  92    object_count = 0
  93    size = 0
  94
  95    checked = {}
  96    rcode = 1
  97
  98    for object_no in range(objects):
  99       if show_pbar:
 100          pbar.display(object_no+1)
 101
 102       object = root_folder.linear[object_no]
 103       object_count = object_count + 1
 104
 105       if object.isBookmark:
 106          if object.href.startswith('place:'): # Firefox SmartBookmarks
 107             log("Skipped %s" % object.href)
 108             continue
 109
 110          if only_errors:
 111             if hasattr(object, "error"):
 112                delattr(object, "error")
 113             else:
 114                continue
 115
 116          if checked.has_key(object.href):
 117             log("Already checked %s" % object.href)
 118             old_object = root_folder.linear[checked[object.href]]
 119             for attr_name in ("last_visit", "last_modified",
 120                   "error", "no_error", "moved", "size", "md5", "real_title",
 121                   "last_tested", "test_time", "icon", "charset"):
 122                if hasattr(old_object, attr_name):
 123                   setattr(object, attr_name, getattr(old_object, attr_name))
 124          else:
 125             log("Checking %s" % object.href)
 126             rcode = robot.check_url(object)
 127
 128             if rcode:
 129                checked[object.href] = object_no
 130                urls_no = urls_no + 1
 131                try:
 132                   size = size + int(object.size)
 133                except (AttributeError, TypeError, ValueError):
 134                   pass # Some object does not have a size :(
 135             else:
 136                log("Interrupted by user (^C)")
 137                break
 138    robot.stop()
 139
 140    if show_pbar:
 141       del pbar
 142
 143    if report_stats:
 144       print "Ok"
 145       print object_count, "objects passed"
 146       print urls_no, "URLs checked"
 147       print size, "bytes eaten"
 148
 149    break_tree(root_folder.linear)
 150    storage.store(root_folder)
 151
 152    if rcode:
 153       log("check_urls finished ok")
 154    log.close()
 155
 156    try:
 157       os.unlink(tempfname)
 158    except os.error:
 159       pass
 160
 161
 162 if __name__ == '__main__':
 163    run()