HTML parser based on BeautifulSoup4

[bookmarks_db.git] / parse_html / bkmk_parse_html.py

diff --git a/parse_html/bkmk_parse_html.py b/parse_html/bkmk_parse_html.py
index 7bc4640d31ccad2a902335c8a2aad978b02bb09c..fa72e1f2aa14f5296343434895bcf6ccb22e088d 100644 (file)
--- a/parse_html/bkmk_parse_html.py
+++ b/parse_html/bkmk_parse_html.py
@@ -18,6 +18,14 @@ DEFAULT_CHARSET = "cp1251" # Stupid default for Russian Cyrillic
 
 parsers = []
 
+try:
+    from . import bkmk_ph_beautifulsoup4
+except ImportError:
+    pass
+else:
+    bkmk_ph_beautifulsoup4.DEFAULT_CHARSET = DEFAULT_CHARSET
+    parsers.append(bkmk_ph_beautifulsoup4.parse_html)
+
 try:
     from . import bkmk_ph_beautifulsoup
 except ImportError:
@@ -136,6 +144,17 @@ def parse_html(html_text, charset=None, log=None):
         p, parser = _parsers[0]
     if log: log("   Using %s" % p.__module__)
 
+    title = parser.title
+    if isinstance(title, unicode):
+        if parser.charset:
+            parser.title = title.encode(parser.charset)
+        else:
+            try:
+                parser.title = title.encode('ascii')
+            except UnicodeEncodeError:
+                parser.charset = DEFAULT_CHARSET
+                parser.title = title.encode(DEFAULT_CHARSET)
+
     converted_title = title = parser.title
     if title and (not parser.charset):
         try: