check_urls.py

   1 #! /usr/bin/env python
   2 """Robot interface - check URLs from bookmarks database
   3
   4 This file is a part of Bookmarks database and Internet robot.
   5 """
   6
   7 from __future__ import print_function
   8 import sys, os
   9
  10
  11 __author__ = "Oleg Broytman <phd@phdru.name>"
  12 __copyright__ = "Copyright (C) 2000-2017 PhiloSoft Design"
  13 __license__ = "GNU GPL"
  14
  15
  16 def run():
  17     from getopt import getopt
  18     optlist, args = getopt(sys.argv[1:], "ise")
  19
  20     show_pbar = 1
  21     report_stats = 1
  22     only_errors = 0
  23
  24     for _opt, _arg in optlist:
  25         if _opt == '-i':
  26             show_pbar = 0
  27         if _opt == '-s':
  28             report_stats = 0
  29         if _opt == '-e':
  30             only_errors = 1
  31     try:
  32         del _opt, _arg
  33     except NameError:
  34         pass
  35
  36     if report_stats:
  37         print("Broytman check_urls, Copyright (C) 1997-2017 PhiloSoft Design")
  38
  39     if args:
  40         sys.stderr.write("check_urls: too many arguments\n")
  41         sys.stderr.write("Usage: check_urls [-ise]\n")
  42         sys.exit(1)
  43
  44     if show_pbar:
  45         show_pbar = sys.stderr.isatty()
  46
  47     if show_pbar:
  48         try:
  49             from m_lib.pbar.tty_pbar import ttyProgressBar
  50         except ImportError:
  51             show_pbar = 0
  52
  53     from m_lib.flog import makelog, openlog
  54     if only_errors:
  55         log = openlog("check.log")
  56         log("chk_urls restarted for errors")
  57         if report_stats:
  58             print("chk_urls restarted for errors")
  59     else:
  60         log = makelog("check.log")
  61         log("check_urls started")
  62         if report_stats:
  63             print("   check_urls: normal start")
  64
  65     from storage import storage
  66     storage = storage()
  67
  68     from robots import robot
  69     robot = robot(log)
  70
  71     if report_stats:
  72         sys.stdout.write("Loading %s: " % storage.filename)
  73         sys.stdout.flush()
  74
  75     root_folder = storage.load()
  76     from bkmk_objects import make_linear, break_tree
  77     make_linear(root_folder)
  78     objects = len(root_folder.linear)
  79
  80     if report_stats:
  81         print("Ok")
  82
  83     if report_stats:
  84         if only_errors:
  85             s = "Rechecking errors: "
  86         else:
  87             s = "Checking: "
  88         sys.stdout.write(s)
  89         sys.stdout.flush()
  90
  91     if show_pbar:
  92         pbar = ttyProgressBar(0, objects)
  93
  94     urls_no = 0
  95     object_count = 0
  96     size = 0
  97
  98     checked = {}
  99     rcode = 1
 100
 101     for object_no in range(objects):
 102         if show_pbar:
 103             pbar.display(object_no+1)
 104
 105         object = root_folder.linear[object_no]
 106         object_count = object_count + 1
 107
 108         if object.isBookmark:
 109             href = object.href
 110             if (href.startswith('place:') # Firefox SmartBookmarks
 111                   or '%s' in href): # Bookmark with keyword
 112                 log("Skipped %s" % href)
 113                 continue
 114
 115             if only_errors:
 116                 if hasattr(object, "error"):
 117                     delattr(object, "error")
 118                 else:
 119                     continue
 120
 121             if checked.has_key(href):
 122                 log("Already checked %s" % href)
 123                 old_object = root_folder.linear[checked[href]]
 124                 for attr_name in ("last_visit", "last_modified",
 125                       "error", "no_error", "moved", "size", "md5", "real_title",
 126                       "last_tested", "test_time", "icon", "charset"):
 127                     if hasattr(old_object, attr_name):
 128                         setattr(object, attr_name, getattr(old_object, attr_name))
 129             else:
 130                 log("Checking %s" % href)
 131                 rcode = robot.check_url(object)
 132
 133                 if rcode:
 134                     checked[href] = object_no
 135                     urls_no = urls_no + 1
 136                     try:
 137                         size = size + int(object.size)
 138                     except (AttributeError, TypeError, ValueError):
 139                         pass # Some object does not have a size :(
 140                 else:
 141                     log("Interrupted by user (^C)")
 142                     break
 143     robot.stop()
 144
 145     if show_pbar:
 146         del pbar
 147
 148     if report_stats:
 149         print("Ok")
 150         print(object_count, "objects passed")
 151         print(urls_no, "URLs checked")
 152         print(size, "bytes eaten")
 153
 154     break_tree(root_folder.linear)
 155     storage.store(root_folder)
 156
 157     if rcode:
 158         log("check_urls finished ok")
 159     log.close()
 160
 161
 162 if __name__ == '__main__':
 163     run()