X-Git-Url: https://git.phdru.name/?a=blobdiff_plain;ds=sidebyside;f=parse_html%2Fbkmk_ph_lxml.py;h=1fa47917deaf81fb6b40dbb413498e45dec7eadc;hb=HEAD;hp=b14be408571ba53e0cfa5f60997394658ad2061b;hpb=80054d3bdcbde67d3b941e0b69b55ca4998f4673;p=bookmarks_db.git

diff --git a/parse_html/bkmk_ph_lxml.py b/parse_html/bkmk_ph_lxml.py
index b14be40..2427482 100644
--- a/parse_html/bkmk_ph_lxml.py
+++ b/parse_html/bkmk_ph_lxml.py
@@ -1,24 +1,44 @@
 """HTML Parser using lxml.html
 
 This file is a part of Bookmarks database and Internet robot.
+
 """
 
 __author__ = "Oleg Broytman <phd@phdru.name>"
-__copyright__ = "Copyright (C) 2010-2012 PhiloSoft Design"
+__copyright__ = "Copyright (C) 2010-2023 PhiloSoft Design"
 __license__ = "GNU GPL"
 
 __all__ = ['parse_html']
 
 
-from lxml.html import parse
+import re
+from lxml.html import fromstring
 from .bkmk_ph_util import HTMLParser
 
 
-def parse_html(filename, charset=None, log=None):
-    html_tree = parse(filename)
-
-    if html_tree.getroot() is None:
+def parse_html(html_text, charset=None, log=None):
+    if not html_text:
         return None
+    try:
+        html_tree = fromstring(html_text)
+    except ValueError as e:
+        if e.args[0].startswith(
+            'Unicode strings with encoding declaration are not supported.'
+            ' Please use bytes input'
+        ):
+            if not charset:
+                match = re.search(
+                    '<\\?xml version="(\\d|.)+" encoding="([^"]+)"\\?>',
+                    html_text, re.U)
+                if match:
+                    charset = match.group(2)
+            if charset:
+                html_text = html_text.encode(charset)
+                html_tree = fromstring(html_text)
+            else:
+                return None
+        else:
+            raise
 
     title = html_tree.findtext('head/title')
     if title is None:
@@ -35,11 +55,14 @@ def parse_html(filename, charset=None, log=None):
                     break
                 except IndexError:
                     meta_charset = False
+        elif m.get('charset', ''):
+            meta_charset = m.get('charset').lower()
+            break
     else:
         meta_charset = False
 
-    if title and (charset or meta_charset):
-        title = title.encode(charset or meta_charset)
+    #if title and (charset or meta_charset):
+    #    title = title.encode(charset or meta_charset)
 
     for m in meta:
         if m.get('http-equiv', '').lower() == 'refresh':