X-Git-Url: https://git.phdru.name/?a=blobdiff_plain;f=parse_html%2Fbkmk_parse_html.py;h=7764303a4e754715a86a3a12e5d0b48d1d5acb8c;hb=3035a4957e210d7a6befcb224f237021b992d285;hp=d885d52fbb3b471b8fb33de93b9e6e9a9dac39e1;hpb=4c235e58fdef35765aedaa40ef106f5ad0fce269;p=bookmarks_db.git

diff --git a/parse_html/bkmk_parse_html.py b/parse_html/bkmk_parse_html.py
index d885d52..7764303 100644
--- a/parse_html/bkmk_parse_html.py
+++ b/parse_html/bkmk_parse_html.py
@@ -14,9 +14,12 @@ __all__ = ['parse_html', 'parse_filename', 'universal_charset']
 import codecs
 import os
 import re
-from htmlentitydefs import name2codepoint
+try:
+    from html.entities import name2codepoint
+except ImportError:
+    from htmlentitydefs import name2codepoint
 
-from ..compat import unicode, unichr
+from compat import unicode, unichr
 
 DEFAULT_CHARSET = "cp1251"  # Stupid default for Russian Cyrillic
 parsers = []
@@ -78,7 +81,7 @@ def recode_entities(title, charset):
               entity_re.match(part):
             _part = name2codepoint.get(part[1:-1], None)
             if _part is not None:
-                part = unichr(_part).encode(charset)
+                part = unichr(_part)
         output.append(part)
     title = ''.join(output)
 
@@ -86,7 +89,7 @@ def recode_entities(title, charset):
     for part in num_entity_re.split(title):
         if num_entity_re.match(part):
             try:
-                part = unichr(int(part[2:-1])).encode(charset)
+                part = unichr(int(part[2:-1]))
             except UnicodeEncodeError:
                 pass  # Leave the entity as is
         output.append(part)
@@ -98,7 +101,7 @@ BKMK_DEBUG_HTML_PARSERS = os.environ.get("BKMK_DEBUG_HTML_PARSERS")
 
 
 def parse_html(html_text, charset=None, log=None):
-    if not parsers:
+    if not html_text or not parsers:
         return None
 
     if charset:
@@ -146,26 +149,26 @@ def parse_html(html_text, charset=None, log=None):
         p, parser = _parsers[0]
     if log: log("   Using %s" % p.__module__)
 
-    title = parser.title
-    if isinstance(title, unicode):
-        if parser.charset:
-            parser.title = title.encode(parser.charset)
-        else:
-            try:
-                parser.title = title.encode('ascii')
-            except UnicodeEncodeError:
-                try:
-                    parser.title = title.encode(DEFAULT_CHARSET)
-                except UnicodeEncodeError:
-                    parser.title = title.encode(universal_charset)
-                    parser.charset = universal_charset
-                else:
-                    parser.charset = DEFAULT_CHARSET
-            else:
-                parser.charset = 'ascii'
+    #title = parser.title
+    #if isinstance(title, unicode):
+    #    if parser.charset:
+    #        parser.title = title.encode(parser.charset)
+    #    else:
+    #        try:
+    #            parser.title = title.encode('ascii')
+    #        except UnicodeEncodeError:
+    #            try:
+    #                parser.title = title.encode(DEFAULT_CHARSET)
+    #            except UnicodeEncodeError:
+    #                parser.title = title.encode(universal_charset)
+    #                parser.charset = universal_charset
+    #            else:
+    #                parser.charset = DEFAULT_CHARSET
+    #        else:
+    #            parser.charset = 'ascii'
 
     converted_title = title = parser.title
-    if title and (not parser.charset):
+    if title and isinstance(title, bytes) and (not parser.charset):
         try:
             title.decode("ascii")
         except UnicodeDecodeError:
@@ -184,21 +187,21 @@ def parse_html(html_text, charset=None, log=None):
                 if log: log("   guessed charset: %s" % parser.charset)
             # if log: log("   current charset: %s" % universal_charset)
             if log: log("   title          : %s" % title)
-            if parser.charset != universal_charset:
-                try:
-                    converted_title = title.decode(parser.charset).\
-                        encode(universal_charset)
-                except UnicodeError:
-                    if log:
-                        log("   incorrect conversion from %s,"
-                            "converting from %s"
-                            % (parser.charset, DEFAULT_CHARSET))
-                    converted_title = \
-                        title.decode(DEFAULT_CHARSET, "replace").\
-                        encode(universal_charset, "replace")
-                    parser.charset = DEFAULT_CHARSET
-            if log and (converted_title != title):
-                log("   converted title: %s" % converted_title)
+            #if parser.charset != universal_charset:
+            #    try:
+            #        converted_title = title.decode(parser.charset).\
+            #            encode(universal_charset)
+            #    except UnicodeError:
+            #        if log:
+            #            log("   incorrect conversion from %s,"
+            #                "converting from %s"
+            #                % (parser.charset, DEFAULT_CHARSET))
+            #        converted_title = \
+            #            title.decode(DEFAULT_CHARSET, "replace").\
+            #            encode(universal_charset, "replace")
+            #        parser.charset = DEFAULT_CHARSET
+            #if log and (converted_title != title):
+            #    log("   converted title: %s" % converted_title)
         except LookupError:
             if log: log("   unknown charset: '%s'" % parser.charset)
     else:
@@ -212,18 +215,18 @@ def parse_html(html_text, charset=None, log=None):
             log("   final title    : %s" % final_title)
         parser.title = final_title
 
-    icon = parser.icon
-    if isinstance(icon, unicode):
-        try:
-            parser.icon = icon.encode('ascii')
-        except UnicodeEncodeError:
-            if parser.charset:
-                parser.icon = icon.encode(parser.charset)
+    #icon = parser.icon
+    #if isinstance(icon, unicode):
+    #    try:
+    #        parser.icon = icon.encode('ascii')
+    #    except UnicodeEncodeError:
+    #        if parser.charset:
+    #            parser.icon = icon.encode(parser.charset)
     return parser
 
 
 def parse_filename(filename, charset=None, log=None):
-    fp = open(filename, 'r')
+    fp = open(filename, 'rt')
     try:
         parser = parse_html(fp.read(), charset=charset, log=log)
     finally: