1 """HTML Parser using html5
3 This file is a part of Bookmarks database and Internet robot.
7 __author__ = "Oleg Broytman <phd@phdru.name>"
8 __copyright__ = "Copyright (C) 2010-2023 PhiloSoft Design"
9 __license__ = "GNU GPL"
11 __all__ = ['parse_html']
14 from html5lib import HTMLParser as HTML5Parser
15 from .bkmk_ph_util import HTMLParser
18 def parse_html(html_text, charset=None, log=None):
21 parser = HTML5Parser()
22 if isinstance(html_text, bytes):
23 html_tree = parser.parse(
24 html_text, encoding=charset, parseMeta=bool(charset))
26 html_tree = parser.parse(html_text)
29 if hasattr(html_tree, 'childNodes'):
30 for node in html_tree.childNodes:
31 # Skip DocType element
32 if (node.name == 'html') and (node.type != 3):
39 for node in html.childNodes:
40 if node.name == 'head':
52 for node in head.childNodes:
53 if node.name == 'title':
55 title = node.childNodes[0].value
60 for node in head.childNodes:
61 if (node.name == 'meta') and \
62 ('http-equiv' in node.attributes) and \
63 (node.attributes['http-equiv'] == 'content-type'):
64 meta_content = node.attributes['content']
68 meta_content.lower().split('charset=')[1].\
74 elif (node.name == 'meta') and ('charset' in node.attributes):
75 meta_charset = node.attributes['charset'].lower()
79 charset = parser.tokenizer.stream.charEncoding[0]
81 #if title and (charset or meta_charset):
82 # title = title.encode(charset or meta_charset)
84 for node in head.childNodes:
85 if node.name == 'meta' and \
86 ('http-equiv' in node.attributes) and \
87 (node.attributes['http-equiv'] == 'refresh'):
88 refresh = node.attributes['content']
91 for node in head.childNodes:
92 if node.name == 'link' and \
93 ('rel' in node.attributes) and \
94 (node.attributes['rel'] in ('icon', 'shortcut icon')):
95 icon = node.attributes['href']
99 for node in html.childNodes:
100 if node.name == 'title':
102 title = node.childNodes[0].value
107 if (title is None) and (refresh is None) and (icon is None):
109 return HTMLParser(charset, meta_charset, title, refresh, icon)