X-Git-Url: https://git.phdru.name/?a=blobdiff_plain;f=Robots%2Fbkmk_rsimple.py;h=871082fe4a0113eddab4435582443fc29db0d068;hb=c9cfb54ef36e77aee7187e4533f063d407073ad8;hp=e01c7fece23cd612362ef862af0d486a580bfcdc;hpb=a62cd94997a22857d27764042a120f21a226e94b;p=bookmarks_db.git

diff --git a/Robots/bkmk_rsimple.py b/Robots/bkmk_rsimple.py
index e01c7fe..871082f 100644
--- a/Robots/bkmk_rsimple.py
+++ b/Robots/bkmk_rsimple.py
@@ -1,9 +1,18 @@
-"""
-   Simple, strightforward robot
+"""Simple, strightforward robot
 
-   Written by Oleg Broytman. Copyright (C) 2000-2010 PhiloSoft Design.
+This file is a part of Bookmarks database and Internet robot.
 """
 
+__version__ = "$Revision$"[11:-2]
+__revision__ = "$Id$"[5:-2]
+__date__ = "$Date$"[7:-2]
+__author__ = "Oleg Broytman <phd@phdru.name>"
+__copyright__ = "Copyright (C) 2000-2011 PhiloSoft Design"
+__license__ = "GNU GPL"
+
+__all__ = ['robot_simple', 'get_error']
+
+
 import sys, os
 import time, urllib
 from base64 import b64encode
@@ -105,9 +114,7 @@ icons = {} # Icon cache; maps URL to a tuple (content type, data)
 
 class robot_simple(Robot):
    def check_url(self, bookmark):
-      if not self.tempfname:
-         self.tempfname = bookmark.tempfname
-
+      fname = None
       try:
          self.start = int(time.time())
          bookmark.icon = None
@@ -120,7 +127,7 @@ class robot_simple(Robot):
          urllib._urlopener.addheaders[2] = ('Referer', "%s://%s%s" % (url_type, url_host, url_path))
 
          if bookmark.charset: urllib._urlopener.addheader('Accept-Charset', bookmark.charset)
-         fname, headers = urllib.urlretrieve("%s://%s%s" % (url_type, url_host, url_path), self.tempfname)
+         fname, headers = urllib.urlretrieve("%s://%s%s" % (url_type, url_host, url_path))
          if bookmark.charset: del urllib._urlopener.addheaders[-1]
 
          size = 0
@@ -152,7 +159,7 @@ class robot_simple(Robot):
          if urllib._urlopener.type == "ftp": # Pass welcome message through MD5
             md5.update(get_welcome())
 
-         md5.md5file(self.tempfname)
+         md5.md5file(fname)
          bookmark.md5 = str(md5)
 
          if headers:
@@ -178,24 +185,6 @@ class robot_simple(Robot):
                   parser = parse_html(fname, charset, self.log)
                   if parser:
                       bookmark.real_title = parser.title
-                      if parser.refresh:
-                         refresh = parser.refresh
-                         try:
-                            url = refresh.split('=', 1)[1]
-                         except IndexError:
-                            url = "self"
-                         try:
-                            timeout = float(refresh.split(';')[0])
-                         except (IndexError, ValueError):
-                            raise RedirectException("html", "Bad redirect to %s (%s)" % (url, refresh))
-                         else:
-                            try:
-                               timeout = int(refresh.split(';')[0])
-                            except ValueError:
-                               pass # float timeout
-                            raise RedirectException("html", "%s (%s sec)" % (url, timeout))
-
-                      # Get favicon.ico
                       icon = parser.icon
                   else:
                      icon = None
@@ -215,7 +204,7 @@ class robot_simple(Robot):
                         _icon = icon
                         for i in range(8):
                            try:
-                              fname, headers = urllib.urlretrieve(_icon)
+                              icon_fname, headers = urllib.urlretrieve(_icon)
                            except RedirectException, e:
                               _icon = e.url
                               self.log("       redirect to : %s" % _icon)
@@ -226,24 +215,43 @@ class robot_simple(Robot):
                      except:
                         etype, emsg, tb = sys.exc_info()
                         self.log("   no icon        : %s %s" % (etype, emsg))
-                        etype = None
-                        emsg = None
-                        tb = None
+                        etype = emsg = tb = None
                         icons[icon] = None
                      else:
                         content_type = headers["Content-Type"]
-                        if content_type.startswith("image/"):
-                           icon_file = open(fname, "rb")
+                        if content_type.startswith("application/") \
+                              or content_type.startswith("image/") \
+                              or content_type.startswith("text/plain"):
+                           icon_file = open(icon_fname, "rb")
                            icon_data = icon_file.read()
                            icon_file.close()
                            bookmark.icon_href = icon
-                           bookmark.icon = "data:%s;base64,%s" % (content_type, b64encode(icon_data))
                            self.log("   got icon       : %s" % content_type)
+                           if content_type.startswith("application/") \
+                                 or content_type.startswith("text/plain"):
+                              self.log("       non-image content type, assume x-icon")
+                              content_type = 'image/x-icon'
+                           bookmark.icon = "data:%s;base64,%s" % (content_type, b64encode(icon_data))
                            icons[icon] = (content_type, bookmark.icon)
                         else:
                            self.log("   no icon        : bad content type '%s'" % content_type)
                            icons[icon] = None
-                        os.remove(fname)
+                  if parser and parser.refresh:
+                     refresh = parser.refresh
+                     try:
+                        url = refresh.split('=', 1)[1]
+                     except IndexError:
+                        url = "self"
+                     try:
+                        timeout = float(refresh.split(';')[0])
+                     except (IndexError, ValueError):
+                        raise RedirectException("html", "Bad redirect to %s (%s)" % (url, refresh))
+                     else:
+                        try:
+                           timeout = int(refresh.split(';')[0])
+                        except ValueError:
+                           pass # float timeout
+                        raise RedirectException("html", "%s (%s sec)" % (url, timeout))
 
             except KeyError, key:
                self.log("   no header: %s" % key)
@@ -275,16 +283,15 @@ class robot_simple(Robot):
          self.log('   Exception: %s' % bookmark.error)
 
       finally:
-         self.finish_check_url(bookmark)
+         self.finish_check_url(bookmark, fname)
 
       # Tested
       return 1
 
-
-   def finish_check_url(self, bookmark):
+   def finish_check_url(self, bookmark, fname=None):
       # Calculate these attributes even in case of an error
-      if os.path.exists(self.tempfname):
-         size = str(os.path.getsize(self.tempfname))
+      if fname and os.path.exists(fname):
+         size = str(os.path.getsize(fname))
          if size[-1] == 'L':
             size = size[:-1]
          bookmark.size = size
@@ -294,3 +301,4 @@ class robot_simple(Robot):
 
       now = int(time.time())
       bookmark.test_time = str(now - start)
+      urllib.urlcleanup()