X-Git-Url: https://git.phdru.name/?a=blobdiff_plain;f=parse_html%2Fbkmk_parse_html.py;h=6afb7f9df735f6a487b6b7537fc23f2c3daa71c2;hb=cb9c36b39ed72cd1fa272130d2bcf162a89c3013;hp=fa72e1f2aa14f5296343434895bcf6ccb22e088d;hpb=11632b7623b2b2e01995f013bc6d8ba01c20cf74;p=bookmarks_db.git

diff --git a/parse_html/bkmk_parse_html.py b/parse_html/bkmk_parse_html.py
index fa72e1f..6afb7f9 100644
--- a/parse_html/bkmk_parse_html.py
+++ b/parse_html/bkmk_parse_html.py
@@ -5,18 +5,14 @@ This file is a part of Bookmarks database and Internet robot.
 """
 
 __author__ = "Oleg Broytman <phd@phdru.name>"
-__copyright__ = "Copyright (C) 1997-2017 PhiloSoft Design"
+__copyright__ = "Copyright (C) 1997-2023 PhiloSoft Design"
 __license__ = "GNU GPL"
 
 __all__ = ['parse_html', 'parse_filename', 'universal_charset']
 
 
 import codecs
-
-universal_charset = "utf-8"
-DEFAULT_CHARSET = "cp1251" # Stupid default for Russian Cyrillic
-
-parsers = []
+import os
 
 try:
     from . import bkmk_ph_beautifulsoup4
@@ -56,19 +52,25 @@ else:
     parsers.append(bkmk_ph_htmlparser.parse_html)
 
 # ElementTidy often segfaults
-#try:
-#   from . import bkmk_ph_etreetidy
-#except ImportError:
-#   pass
-#else:
-#   parsers.append(bkmk_ph_etreetidy.parse_html)
+# try:
+#     from . import bkmk_ph_etreetidy
+# except ImportError:
+#     pass
+# else:
+#     parsers.append(bkmk_ph_etreetidy.parse_html)
 
 import re
 from htmlentitydefs import name2codepoint
 
+universal_charset = "utf-8"
+DEFAULT_CHARSET = "cp1251"  # Stupid default for Russian Cyrillic
+
+parsers = []
+
 entity_re = re.compile("(&\w+;)")
 num_entity_re = re.compile("(&#[0-9]+;)")
 
+
 def recode_entities(title, charset):
     output = []
     for part in entity_re.split(title):
@@ -86,22 +88,22 @@ def recode_entities(title, charset):
             try:
                 part = unichr(int(part[2:-1])).encode(charset)
             except UnicodeEncodeError:
-                pass # Leave the entity as is
+                pass  # Leave the entity as is
         output.append(part)
 
     return ''.join(output)
 
 
-import os
 BKMK_DEBUG_HTML_PARSERS = os.environ.get("BKMK_DEBUG_HTML_PARSERS")
 
+
 def parse_html(html_text, charset=None, log=None):
     if not parsers:
         return None
 
     if charset:
         try:
-            codecs.lookup(charset) # In case of unknown charset...
+            codecs.lookup(charset)  # In case of unknown charset...
         except (ValueError, LookupError):
             charset = None         # ...try charset from HTML
 
@@ -152,8 +154,15 @@ def parse_html(html_text, charset=None, log=None):
             try:
                 parser.title = title.encode('ascii')
             except UnicodeEncodeError:
-                parser.charset = DEFAULT_CHARSET
-                parser.title = title.encode(DEFAULT_CHARSET)
+                try:
+                    parser.title = title.encode(DEFAULT_CHARSET)
+                except UnicodeEncodeError:
+                    parser.title = title.encode(universal_charset)
+                    parser.charset = universal_charset
+                else:
+                    parser.charset = DEFAULT_CHARSET
+            else:
+                parser.charset = 'ascii'
 
     converted_title = title = parser.title
     if title and (not parser.charset):
@@ -173,16 +182,23 @@ def parse_html(html_text, charset=None, log=None):
                 if log: log("   META charset   : %s" % parser.charset)
             elif (not charset) or (charset != parser.charset):
                 if log: log("   guessed charset: %s" % parser.charset)
-            #if log: log("   current charset: %s" % universal_charset)
+            # if log: log("   current charset: %s" % universal_charset)
             if log: log("   title          : %s" % title)
             if parser.charset != universal_charset:
                 try:
-                    converted_title = unicode(title, parser.charset).encode(universal_charset)
+                    converted_title = unicode(title, parser.charset).\
+                        encode(universal_charset)
                 except UnicodeError:
-                    if log: log("   incorrect conversion from %s, converting from %s" % (parser.charset, DEFAULT_CHARSET))
-                    converted_title = unicode(title, DEFAULT_CHARSET, "replace").encode(universal_charset, "replace")
+                    if log:
+                        log("   incorrect conversion from %s,"
+                            "converting from %s"
+                            % (parser.charset, DEFAULT_CHARSET))
+                    converted_title = \
+                        unicode(title, DEFAULT_CHARSET, "replace").\
+                        encode(universal_charset, "replace")
                     parser.charset = DEFAULT_CHARSET
-            if log and (converted_title != title): log("   converted title: %s" % converted_title)
+            if log and (converted_title != title):
+                log("   converted title: %s" % converted_title)
         except LookupError:
             if log: log("   unknown charset: '%s'" % parser.charset)
     else:
@@ -192,7 +208,8 @@ def parse_html(html_text, charset=None, log=None):
         final_title = recode_entities(converted_title, universal_charset)
         parts = [s.strip() for s in final_title.replace('\r', '').split('\n')]
         final_title = ' '.join([s for s in parts if s])
-        if log and (final_title != converted_title): log("   final title    : %s" % final_title)
+        if log and (final_title != converted_title):
+            log("   final title    : %s" % final_title)
         parser.title = final_title
 
     icon = parser.icon
@@ -204,6 +221,7 @@ def parse_html(html_text, charset=None, log=None):
                 parser.icon = icon.encode(parser.charset)
     return parser
 
+
 def parse_filename(filename, charset=None, log=None):
     fp = open(filename, 'r')
     try: