1 """HTML Parser using ElementTree+TidyLib.
3 This file is a part of Bookmarks database and Internet robot.
7 __author__ = "Oleg Broytman <phd@phdru.name>"
8 __copyright__ = "Copyright (C) 2010-2014 PhiloSoft Design"
9 __license__ = "GNU GPL"
11 __all__ = ['parse_html']
14 from elementtidy import TidyHTMLTreeBuilder
15 from .bkmk_ph_util import HTMLParser
18 def parse_html(html_text, charset=None, log=None):
20 html_tree = TidyHTMLTreeBuilder.parseString(html_text)
24 XHTML = "{http://www.w3.org/1999/xhtml}"
26 for elem in html_tree.getiterator():
27 if elem.tag.startswith(XHTML):
28 elem.tag = elem.tag[len(XHTML):]
30 title = html_tree.findtext('head/title')
32 title = html_tree.findtext('title')
34 meta = html_tree.findall('head/meta')
36 if m.get('http-equiv', '').lower() == 'content-type':
37 meta_content = m.get("content")
41 meta_content.lower().split('charset=')[1].split(';')[0]
45 elif m.get('charset', ''):
46 meta_charset = m.get('charset').lower()
51 if title and (charset or meta_charset):
52 title = title.encode(charset or meta_charset)
55 if m.get('http-equiv', '').lower() == 'refresh':
56 refresh = m.get("content")
61 for link in html_tree.findall('head/link'):
62 if link.get('rel', '').lower() in ('icon', 'shortcut icon'):
63 icon = link.get("href")
68 if (title is None) and (refresh is None) and (icon is None):
70 return HTMLParser(charset, meta_charset, title, refresh, icon)