Better fake headers.

[bookmarks_db.git] / Robots / bkmk_rsimple.py

diff --git a/Robots/bkmk_rsimple.py b/Robots/bkmk_rsimple.py
index 14a3f5804799365ebd0db637f8ecb04a3b593df9..399cf064900be84c7a29e2d43ded5df7aeb7dd47 100644 (file)
--- a/Robots/bkmk_rsimple.py
+++ b/Robots/bkmk_rsimple.py
@@ -4,14 +4,17 @@
    Written by Oleg BroytMann. Copyright (C) 2000-2007 PhiloSoft Design.
 """
 
-
 import sys, os
 import time, urllib
 from base64 import b64encode
 from urlparse import urljoin
+
 from m_lib.net.www.util import parse_time
 from m_lib.md5wrapper import md5wrapper
 
+from bkmk_objects import Robot
+from parse_html import parse_html
+
 
 class RedirectException(Exception):
    reloc_dict = {
@@ -46,18 +49,17 @@ class MyURLopener(urllib.URLopener):
 urllib._urlopener = MyURLopener()
 
 # Fake headers to pretend this is a real browser
-_version = "Links (2.1; Linux 2.6 i686; 80x25)"
+_version = "Mozilla/5.0 (X11; U; Linux 2.6 i686; en) Gecko/20001221 Firefox/2.0.0"
 urllib._urlopener.addheaders[0] = ('User-Agent', _version)
 _version = "bookmarks_db (Python %d.%d.%d; urllib/%s)" % (
    sys.version_info[0], sys.version_info[1], sys.version_info[2], urllib.__version__)
 urllib._urlopener.addheader('X-User-Agent', _version)
+urllib._urlopener.addheader('Referer', '')
 
 urllib._urlopener.addheader('Connection', 'close')
-urllib._urlopener.addheader('Content-Length', '0')
 urllib._urlopener.addheader('Accept', '*/*')
 urllib._urlopener.addheader('Accept-Language', 'ru,en')
 urllib._urlopener.addheader('Cache-Control', 'max-age=300')
-urllib._urlopener.addheader('Referer', 'http://www.yahoo.com/')
 
 
 def get_error(msg):
@@ -90,9 +92,6 @@ def get_welcome():
    return _welcome
 
 
-from bkmk_objects import Robot
-from parse_html import parse_html
-
 class robot_simple(Robot):
    def check_url(self, bookmark):
       if not self.tempfname:
@@ -107,6 +106,7 @@ class robot_simple(Robot):
             url_host, url_path = urllib.splithost(url_rest)
             url_path, url_tag  = urllib.splittag(url_path)
 
+            urllib._urlopener.addheaders[2] = ('Referer', "%s://%s%s" % (url_type, url_host, url_path))
             if bookmark.charset: urllib._urlopener.addheader('Accept-Charset', bookmark.charset)
             fname, headers = urllib.urlretrieve("%s://%s%s" % (url_type, url_host, url_path), self.tempfname)
             if bookmark.charset: del urllib._urlopener.addheaders[-1]
@@ -150,10 +150,10 @@ class robot_simple(Robot):
                      content_type, charset = content_type.split(';')
                      content_type = content_type.strip()
                      charset = charset.split('=')[1].strip()
-                     if self.log: self.log("   HTTP charset   : %s" % charset)
+                     self.log("   HTTP charset   : %s" % charset)
                   except (ValueError, IndexError):
                      charset = None
-                     if self.log: self.log("   no charset in Content-Type header")
+                     self.log("   no charset in Content-Type header")
                   if content_type == "text/html":
                      parser = parse_html(fname, charset, self.log)
                      bookmark.real_title = parser.title
@@ -179,21 +179,21 @@ class robot_simple(Robot):
                      if not icon:
                         icon = "/favicon.ico"
                      icon = urljoin("%s://%s%s" % (url_type, url_host, url_path), icon)
-                     if self.log: self.log("   looking for icon at: %s" % icon)
+                     self.log("   looking for icon at: %s" % icon)
                      try:
                         for i in range(8):
                            try:
                               fname, headers = urllib.urlretrieve(icon)
                            except RedirectException, e:
                               icon = e.url
-                              if self.log: self.log("       redirect to : %s" % icon)
+                              self.log("       redirect to : %s" % icon)
                            else:
                               break
                         else:
                            raise IOError("Too many redirects")
                      except:
                         etype, emsg, tb = sys.exc_info()
-                        if self.log: self.log("   no icon        : %s %s" % (etype, emsg))
+                        self.log("   no icon        : %s %s" % (etype, emsg))
                         etype = None
                         emsg = None
                         tb = None
@@ -204,9 +204,9 @@ class robot_simple(Robot):
                            icon = icon_file.read()
                            icon_file.close()
                            bookmark.icon = "data:%s;base64,%s" % (content_type, b64encode(icon))
-                           if self.log: self.log("   got icon       : %s" % content_type)
+                           self.log("   got icon       : %s" % content_type)
                         else:
-                           if self.log: self.log("   no icon        : bad content type '%s'" % content_type)
+                           self.log("   no icon        : bad content type '%s'" % content_type)
                         os.remove(fname)
 
                except KeyError: