X-Git-Url: https://git.phdru.name/?a=blobdiff_plain;f=Robots%2Fbkmk_robot_base.py;h=85c704a64955608c5add79747020d2b48ea74889;hb=48440e20170112c8a036b0c66c1cbe067ed1d87e;hp=3b5ec274760a59b3578e25a5bec87a7b7d09c7ac;hpb=2e673dc1e4202710ca17ee2f6eab2dae721139ac;p=bookmarks_db.git

diff --git a/Robots/bkmk_robot_base.py b/Robots/bkmk_robot_base.py
index 3b5ec27..85c704a 100644
--- a/Robots/bkmk_robot_base.py
+++ b/Robots/bkmk_robot_base.py
@@ -15,8 +15,11 @@ from base64 import b64encode
 import sys
 import socket
 import time
-import urllib
-from urlparse import urljoin
+try:
+    from urllib.parse import splittype, splithost, splittag, urljoin
+except ImportError:
+    from urllib import splittype, splithost, splittag
+    from urlparse import urljoin
 
 from m_lib.md5wrapper import md5wrapper
 from m_lib.net.www.util import parse_time
@@ -62,10 +65,10 @@ class robot_base(Robot):
             self.start = int(time.time())
             bookmark.icon = None
 
-            url_type, url_rest = urllib.splittype(bookmark.href)
-            url_host, url_path = urllib.splithost(url_rest)
-            url_path, url_tag  = urllib.splittag(url_path)  # noqa: E221
-            #                            multiple spaces before operator
+            url_type, url_rest = splittype(bookmark.href)
+            url_host, url_path = splithost(url_rest)
+            url_path, url_tag  = splittag(url_path)  # noqa: E221
+            #                    multiple spaces before operator
 
             url = "%s://%s%s" % (url_type, url_host, url_path)
             error, redirect_code, redirect_to, headers, content = \
@@ -108,15 +111,28 @@ class robot_base(Robot):
 
             md5 = md5wrapper()
             if url_type == "ftp":  # Pass welcome message through MD5
-                md5.update(self.get_ftp_welcome())
+                ftp_welcome = self.get_ftp_welcome()
+                if not isinstance(ftp_welcome, bytes):
+                    ftp_welcome = ftp_welcome.encode('utf-8')
+                md5.update(ftp_welcome)
 
-            md5.update(content)
+            if isinstance(content, bytes):
+                md5.update(content)
+            else:
+                md5.update(content.encode('utf-8'))
             bookmark.md5 = str(md5)
 
             if headers:
                 try:
                     content_type = headers["Content-Type"]
                     self.log("   Content-Type: %s" % content_type)
+                    if content_type is None:
+                        if 'html' in content.lower():
+                            content_type = 'text/html'
+                        else:
+                            content_type = 'text/plain'
+                        self.log("   Set Content-Type to: %s"
+                                 % content_type)
                     try:
                         # extract charset from
                         # "text/html; foo; charset=UTF-8, bar; baz;"
@@ -178,9 +194,11 @@ class robot_base(Robot):
                                 icons[icon_url] = None
                             else:
                                 content_type = icon_headers["Content-Type"]
-                                if content_type.startswith("application/") \
-                                   or content_type.startswith("image/") \
-                                   or content_type.startswith("text/plain"):
+                                if content_type and (
+                                    content_type.startswith("application/")
+                                    or content_type.startswith("image/")
+                                    or content_type.startswith("text/plain")
+                                ):
                                     bookmark.icon_href = icon_url
                                     self.log("   got icon       : %s"
                                              % content_type)
@@ -192,6 +210,8 @@ class robot_base(Robot):
                                         self.log("   non-image content type,"
                                                  " assume x-icon")
                                         content_type = 'image/x-icon'
+                                    if not isinstance(icon_data, bytes):
+                                        icon_data = icon_data.encode('utf-8')
                                     bookmark.icon = "data:%s;base64,%s" \
                                         % (content_type, b64encode(icon_data))
                                     icons[icon_url] = (content_type,
@@ -254,8 +274,15 @@ class robot_base(Robot):
         return 1
 
     def set_redirect(self, bookmark, errcode, newurl):
-        bookmark.moved = "(%s) to %s" % (reloc_dict[errcode], newurl)
-        self.log('   Moved: %s' % bookmark.moved)
+        bookmark.moved = moved = "(%s) to %s" % (reloc_dict[errcode], newurl)
+        try:
+            moved.encode('ascii')
+        except UnicodeEncodeError:
+            try:
+                moved = moved.encode(bookmark.charset)
+            except (LookupError, TypeError, UnicodeEncodeError):
+                moved = moved.encode('utf-8')
+        self.log('   Moved: %s' % moved)
 
     def finish_check_url(self, bookmark):
         start = self.start