"""HTML Parser using lxml.html
This file is a part of Bookmarks database and Internet robot.
+
"""
__author__ = "Oleg Broytman <phd@phdru.name>"
-__copyright__ = "Copyright (C) 2010-2012 PhiloSoft Design"
+__copyright__ = "Copyright (C) 2010-2023 PhiloSoft Design"
__license__ = "GNU GPL"
__all__ = ['parse_html']
-from lxml.html import parse
+import re
+from lxml.html import fromstring
from .bkmk_ph_util import HTMLParser
-def parse_html(filename, charset=None, log=None):
- html_tree = parse(filename)
-
- if html_tree.getroot() is None:
+def parse_html(html_text, charset=None, log=None):
+ if not html_text:
return None
+ try:
+ html_tree = fromstring(html_text)
+ except ValueError as e:
+ if e.args[0].startswith(
+ 'Unicode strings with encoding declaration are not supported.'
+ ' Please use bytes input'
+ ):
+ if not charset:
+ match = re.search(
+ '<\\?xml version="(\\d|.)+" encoding="([^"]+)"\\?>',
+ html_text, re.U)
+ if match:
+ charset = match.group(2)
+ if charset:
+ html_text = html_text.encode(charset)
+ html_tree = fromstring(html_text)
+ else:
+ return None
+ else:
+ raise
title = html_tree.findtext('head/title')
if title is None:
break
except IndexError:
meta_charset = False
+ elif m.get('charset', ''):
+ meta_charset = m.get('charset').lower()
+ break
else:
meta_charset = False
- if title and (charset or meta_charset):
- title = title.encode(charset or meta_charset)
+ #if title and (charset or meta_charset):
+ # title = title.encode(charset or meta_charset)
for m in meta:
if m.get('http-equiv', '').lower() == 'refresh':