1 """HTML Parser using html5
3 This file is a part of Bookmarks database and Internet robot.
6 __author__ = "Oleg Broytman <phd@phdru.name>"
7 __copyright__ = "Copyright (C) 2010-2012 PhiloSoft Design"
8 __license__ = "GNU GPL"
10 __all__ = ['parse_html']
13 from html5lib import HTMLParser as HTML5Parser
14 from .bkmk_ph_util import HTMLParser
17 def parse_html(filename, charset=None, log=None):
18 parser = HTML5Parser()
20 parser._parse(fp, encoding=charset, parseMeta=bool(charset))
22 html_tree = parser.tree.getDocument()
24 for node in html_tree.childNodes:
25 if (node.name == 'html') and (node.type != 3): # Skip DocType element
34 for node in html.childNodes:
35 if node.name == 'head':
47 for node in head.childNodes:
48 if node.name == 'title':
50 title = node.childNodes[0].value
58 for node in head.childNodes:
59 if node.name == 'meta' and \
60 ('http-equiv' in node.attributes) and \
61 (node.attributes['http-equiv'] == 'content-type'):
62 meta_content = node.attributes['content']
66 meta_content.lower().split('charset=')[1].split(';')[0]
73 charset = parser.tokenizer.stream.charEncoding[0]
75 if charset or meta_charset:
76 title = title.encode(charset or meta_charset)
78 for node in head.childNodes:
79 if node.name == 'meta' and \
80 ('http-equiv' in node.attributes) and \
81 (node.attributes['http-equiv'] == 'refresh'):
82 refresh = node.attributes['content']
85 for node in head.childNodes:
86 if node.name == 'link' and \
87 ('rel' in node.attributes) and \
88 (node.attributes['rel'] in ('icon', 'shortcut icon')):
89 icon = node.attributes['href']
93 for node in html.childNodes:
94 if node.name == 'title':
96 title = node.childNodes[0].value
104 return HTMLParser(charset, meta_charset, title, refresh, icon)