check_urls.py

   1 #! /usr/bin/env python
   2 """
   3    Robot interface - check URLs from bookmarks database
   4
   5    Written by Broytman. Copyright (C) 2000-2010 PhiloSoft Design.
   6 """
   7
   8
   9 import sys, os
  10
  11
  12 def run():
  13    from getopt import getopt
  14    optlist, args = getopt(sys.argv[1:], "ise")
  15
  16    show_pbar = 1
  17    report_stats = 1
  18    only_errors = 0
  19
  20    for _opt, _arg in optlist:
  21       if _opt == '-i':
  22          show_pbar = 0
  23       if _opt == '-s':
  24          report_stats = 0
  25       if _opt == '-e':
  26          only_errors = 1
  27    try:
  28       del _opt, _arg
  29    except NameError:
  30       pass
  31
  32    if report_stats:
  33       print "Broytman check_urls, Copyright (C) 1997-2010 PhiloSoft Design"
  34
  35    if args:
  36       sys.stderr.write("check_urls: too many arguments\n")
  37       sys.stderr.write("Usage: check_urls [-ise]\n")
  38       sys.exit(1)
  39
  40    if show_pbar:
  41       show_pbar = sys.stderr.isatty()
  42
  43    if show_pbar:
  44       try:
  45          from m_lib.pbar.tty_pbar import ttyProgressBar
  46       except ImportError:
  47          show_pbar = 0
  48
  49    from m_lib.flog import makelog, openlog
  50    if only_errors:
  51       log = openlog("check.log")
  52       log("chk_urls restarted for errors")
  53       if report_stats:
  54          print "chk_urls restarted for errors"
  55    else:
  56       log = makelog("check.log")
  57       log("check_urls started")
  58       if report_stats:
  59          print "   check_urls: normal start"
  60
  61    from storage import storage
  62    storage = storage()
  63
  64    from robots import robot
  65    robot = robot(log)
  66
  67    if report_stats:
  68       sys.stdout.write("Loading %s: " % storage.filename)
  69       sys.stdout.flush()
  70
  71    root_folder = storage.load()
  72    from bkmk_objects import make_linear, break_tree
  73    make_linear(root_folder)
  74    objects = len(root_folder.linear)
  75
  76    if report_stats:
  77       print "Ok"
  78
  79    if report_stats:
  80       if only_errors:
  81          s = "Rechecking errors: "
  82       else:
  83          s = "Checking: "
  84       sys.stdout.write(s)
  85       sys.stdout.flush()
  86
  87    if show_pbar:
  88       pbar = ttyProgressBar(0, objects)
  89
  90    urls_no = 0
  91    object_count = 0
  92    size = 0
  93
  94    checked = {}
  95    rcode = 1
  96
  97    for object_no in range(objects):
  98       if show_pbar:
  99          pbar.display(object_no+1)
 100
 101       object = root_folder.linear[object_no]
 102       object_count = object_count + 1
 103
 104       if object.isBookmark:
 105          if object.href.startswith('place:'): # Firefox SmartBookmarks
 106             log("Skipped %s" % object.href)
 107             continue
 108
 109          if only_errors:
 110             if hasattr(object, "error"):
 111                delattr(object, "error")
 112             else:
 113                continue
 114
 115          if checked.has_key(object.href):
 116             log("Already checked %s" % object.href)
 117             old_object = root_folder.linear[checked[object.href]]
 118             for attr_name in ("last_visit", "last_modified",
 119                   "error", "no_error", "moved", "size", "md5", "real_title",
 120                   "last_tested", "test_time", "icon", "charset"):
 121                if hasattr(old_object, attr_name):
 122                   setattr(object, attr_name, getattr(old_object, attr_name))
 123          else:
 124             log("Checking %s" % object.href)
 125             rcode = robot.check_url(object)
 126
 127             if rcode:
 128                checked[object.href] = object_no
 129                urls_no = urls_no + 1
 130                try:
 131                   size = size + int(object.size)
 132                except (AttributeError, TypeError, ValueError):
 133                   pass # Some object does not have a size :(
 134             else:
 135                log("Interrupted by user (^C)")
 136                break
 137    robot.stop()
 138
 139    if show_pbar:
 140       del pbar
 141
 142    if report_stats:
 143       print "Ok"
 144       print object_count, "objects passed"
 145       print urls_no, "URLs checked"
 146       print size, "bytes eaten"
 147
 148    break_tree(root_folder.linear)
 149    storage.store(root_folder)
 150
 151    if rcode:
 152       log("check_urls finished ok")
 153    log.close()
 154
 155
 156 if __name__ == '__main__':
 157    run()