X-Git-Url: https://git.phdru.name/?a=blobdiff_plain;f=Robots%2Fbkmk_robot_base.py;h=3fd567cda2f0c232017d910e4a66526efcec9d75;hb=f6ecaa9dd00b1b897c3c9ffa1a655864f24b7af8;hp=724391853f57f4d7912f579e261d403de35ccb47;hpb=bd078d376a721b31918b60c41ebf15be408bf52a;p=bookmarks_db.git

diff --git a/Robots/bkmk_robot_base.py b/Robots/bkmk_robot_base.py
index 7243918..3fd567c 100644
--- a/Robots/bkmk_robot_base.py
+++ b/Robots/bkmk_robot_base.py
@@ -24,6 +24,22 @@ from bkmk_objects import Robot
 from parse_html import parse_html
 
 
+# Fake headers to pretend this is a real browser
+_user_agent = "Mozilla/5.0 (X11; U; Linux 2.6 i686; en)"
+" Gecko/20001221 Firefox/2.0.0"
+_x_user_agent = "bookmarks_db (Python %d.%d.%d)" % sys.version_info[:3]
+
+request_headers = {
+    'Accept': '*/*',
+    'Accept-Language': 'ru,en',
+    'Cache-Control': 'max-age=300',
+    'Connection': 'close',
+    'Referer': '/',
+    'User-Agent': _user_agent,
+    'X-User-Agent': _x_user_agent,
+}
+
+
 reloc_dict = {
   301: "perm1.",
   302: "temp2.",
@@ -66,7 +82,10 @@ class robot_base(Robot):
             url_type, netloc, url_path, query, url_tag = split_results
             url_host = split_results.hostname
 
-            url = "%s://%s%s" % (url_type, url_host, url_path)
+            if query:
+                url = "%s://%s%s?%s" % (url_type, url_host, url_path, query)
+            else:
+                url = "%s://%s%s" % (url_type, url_host, url_path)
             error, redirect_code, redirect_to, headers, content = \
                 self.get(bookmark, url, True)
 
@@ -135,8 +154,13 @@ class robot_base(Robot):
                             break
                     content_stripped = content.strip()
                     if content_stripped and charset:
-                        content_stripped = content_stripped.decode(
-                            charset, 'replace')
+                        try:
+                            content_stripped = content_stripped.decode(
+                                charset, 'replace')
+                        except LookupError:
+                            charset = None
+                            self.log("   unknown charset "
+                                     "in Content-Type header")
                     if content_stripped and is_html:
                         parser = parse_html(
                             content_stripped, charset, self.log)
@@ -158,9 +182,16 @@ class robot_base(Robot):
                             if icons[icon_url]:
                                 bookmark.icon_href = icon_url
                                 content_type, bookmark.icon = icons[icon_url]
-                                self.log("   cached icon: %s" % content_type)
+                                self.log("   cached icon    : %s"
+                                         % content_type)
                             else:
-                                self.log("   cached icon: no icon")
+                                self.log("   cached icon    : no icon")
+                        elif icon_url.startswith('data:'):
+                            content_type, icon_data = \
+                                icon_url[len('data:'):].split(',', 1)
+                            bookmark.icon_href = bookmark.icon = icon_url
+                            self.log("   got data icon  : %s" % content_type)
+                            icons[icon_url] = (content_type, icon_url)
                         else:
                             try:
                                 _icon_url = icon_url