X-Git-Url: https://git.phdru.name/?a=blobdiff_plain;f=Robots%2Fbkmk_rsimple.py;h=e01c7fece23cd612362ef862af0d486a580bfcdc;hb=1666eb3f5d14ca9b798782ae1de99e9f2c0ae611;hp=0dee51e8837ae3db9922704d78bd3e08b7325df7;hpb=39c6f1c85c93c021fca528bb1a5cca91f53c196c;p=bookmarks_db.git

diff --git a/Robots/bkmk_rsimple.py b/Robots/bkmk_rsimple.py
index 0dee51e..e01c7fe 100644
--- a/Robots/bkmk_rsimple.py
+++ b/Robots/bkmk_rsimple.py
@@ -1,7 +1,7 @@
 """
    Simple, strightforward robot
 
-   Written by Oleg BroytMann. Copyright (C) 2000-2007 PhiloSoft Design.
+   Written by Oleg Broytman. Copyright (C) 2000-2010 PhiloSoft Design.
 """
 
 import sys, os
@@ -40,6 +40,8 @@ class MyURLopener(urllib.URLopener):
 
    # Error 301 -- also relocated (permanently)
    http_error_301 = http_error_302
+   # Error 307 -- also relocated (temporary)
+   http_error_307 = http_error_302
 
    # Error 401 -- authentication required
    def http_error_401(self, url, fp, errcode, errmsg, headers, data=None): 
@@ -156,42 +158,54 @@ class robot_simple(Robot):
          if headers:
             try:
                content_type = headers["Content-Type"]
+               self.log("   Content-Type: %s" % content_type)
                try:
+                  # extract charset from "text/html; foo; charset=UTF-8, bar; baz;"
                   content_type, charset = content_type.split(';', 1)
                   content_type = content_type.strip()
-                  charset = charset.split('=')[1].strip()
+                  charset = charset.split('=')[1].strip().split(',')[0]
                   self.log("   HTTP charset   : %s" % charset)
                except (ValueError, IndexError):
                   charset = None
                   self.log("   no charset in Content-Type header")
-               if content_type in ("text/html", "application/xhtml+xml"):
+               for ctype in ("text/html", "application/xhtml+xml"):
+                  if content_type.startswith(ctype):
+                      html = True
+                      break
+               else:
+                  html = False
+               if html:
                   parser = parse_html(fname, charset, self.log)
-                  bookmark.real_title = parser.title
-                  if parser.refresh:
-                     refresh = parser.refresh
-                     try:
-                        url = refresh.split('=', 1)[1]
-                     except IndexError:
-                        url = "self"
-                     try:
-                        timeout = float(refresh.split(';')[0])
-                     except (IndexError, ValueError):
-                        raise RedirectException("html", "Bad redirect to %s (%s)" % (url, refresh))
-                     else:
-                        try:
-                           timeout = int(refresh.split(';')[0])
-                        except ValueError:
-                           pass # float timeout
-                        raise RedirectException("html", "%s (%s sec)" % (url, timeout))
-
-                  # Get favicon.ico
-                  icon = parser.icon
+                  if parser:
+                      bookmark.real_title = parser.title
+                      if parser.refresh:
+                         refresh = parser.refresh
+                         try:
+                            url = refresh.split('=', 1)[1]
+                         except IndexError:
+                            url = "self"
+                         try:
+                            timeout = float(refresh.split(';')[0])
+                         except (IndexError, ValueError):
+                            raise RedirectException("html", "Bad redirect to %s (%s)" % (url, refresh))
+                         else:
+                            try:
+                               timeout = int(refresh.split(';')[0])
+                            except ValueError:
+                               pass # float timeout
+                            raise RedirectException("html", "%s (%s sec)" % (url, timeout))
+
+                      # Get favicon.ico
+                      icon = parser.icon
+                  else:
+                     icon = None
                   if not icon:
                      icon = "/favicon.ico"
                   icon = urljoin("%s://%s%s" % (url_type, url_host, url_path), icon)
                   self.log("   looking for icon at: %s" % icon)
                   if icon in icons:
                      if icons[icon]:
+                        bookmark.icon_href = icon
                         content_type, bookmark.icon = icons[icon]
                         self.log("       cached icon: %s" % content_type)
                      else:
@@ -220,9 +234,10 @@ class robot_simple(Robot):
                         content_type = headers["Content-Type"]
                         if content_type.startswith("image/"):
                            icon_file = open(fname, "rb")
-                           icon = icon_file.read()
+                           icon_data = icon_file.read()
                            icon_file.close()
-                           bookmark.icon = "data:%s;base64,%s" % (content_type, b64encode(icon))
+                           bookmark.icon_href = icon
+                           bookmark.icon = "data:%s;base64,%s" % (content_type, b64encode(icon_data))
                            self.log("   got icon       : %s" % content_type)
                            icons[icon] = (content_type, bookmark.icon)
                         else:
@@ -230,28 +245,34 @@ class robot_simple(Robot):
                            icons[icon] = None
                         os.remove(fname)
 
-            except KeyError:
-               pass
+            except KeyError, key:
+               self.log("   no header: %s" % key)
 
       except IOError, msg:
          if (msg[0] == "http error") and (msg[1] == -1):
             bookmark.no_error = "The server did not return any header - it is not an error, actually"
+            self.log('   no headers: %s' % bookmark.no_error)
          else:
             bookmark.error = get_error(msg)
+            self.log('   Error: %s' % bookmark.error)
 
       except EOFError:
          bookmark.error = "Unexpected EOF (FTP server closed connection)"
+         self.log('   EOF: %s' % bookmark.error)
 
       except RedirectException, msg:
          bookmark.moved = str(msg)
+         self.log('   Moved: %s' % bookmark.moved)
 
       except KeyboardInterrupt:
+         self.log("Keyboard interrupt (^C)")
          return 0
 
       except:
          import traceback
          traceback.print_exc()
          bookmark.error = "Exception!"
+         self.log('   Exception: %s' % bookmark.error)
 
       finally:
          self.finish_check_url(bookmark)