1 """HTML Parser using lxml.html
3 This file is a part of Bookmarks database and Internet robot.
7 __author__ = "Oleg Broytman <phd@phdru.name>"
8 __copyright__ = "Copyright (C) 2010-2023 PhiloSoft Design"
9 __license__ = "GNU GPL"
11 __all__ = ['parse_html']
15 from lxml.html import fromstring
16 from .bkmk_ph_util import HTMLParser
19 def parse_html(html_text, charset=None, log=None):
21 html_tree = fromstring(html_text)
22 except ValueError as e:
23 if e.args[0].startswith(
24 'Unicode strings with encoding declaration are not supported.'
25 ' Please use bytes input'
29 '<\\?xml version="(\\d|.)+" encoding="([^"]+)"\\?>',
32 charset = match.group(2)
34 html_text = html_text.encode(charset)
35 html_tree = fromstring(html_text)
41 title = html_tree.findtext('head/title')
43 title = html_tree.findtext('title')
45 meta = html_tree.findall('head/meta')
47 if m.get('http-equiv', '').lower() == 'content-type':
48 meta_content = m.get("content")
52 meta_content.lower().split('charset=')[1].split(';')[0]
56 elif m.get('charset', ''):
57 meta_charset = m.get('charset').lower()
62 #if title and (charset or meta_charset):
63 # title = title.encode(charset or meta_charset)
66 if m.get('http-equiv', '').lower() == 'refresh':
67 refresh = m.get("content")
72 for link in html_tree.findall('head/link'):
73 if link.get('rel', '').lower() in ('icon', 'shortcut icon'):
74 icon = link.get("href")
79 if (title is None) and (refresh is None) and (icon is None):
81 return HTMLParser(charset, meta_charset, title, refresh, icon)