1 """HTML Parser using Pythons' HTMLParser
3 This file is a part of Bookmarks database and Internet robot.
7 __author__ = "Oleg Broytman <phd@phdru.name>"
8 __copyright__ = "Copyright (C) 1997-2014 PhiloSoft Design"
9 __license__ = "GNU GPL"
11 __all__ = ['parse_html']
14 from HTMLParser import HTMLParseError
15 from m_lib.net.www.html import HTMLParser as _HTMLParser
18 class HTMLHeadDone(Exception): pass
21 class HTMLParser(_HTMLParser):
22 def __init__(self, charset=None):
23 _HTMLParser.__init__(self)
24 self.charset = charset
33 def do_meta(self, attrs):
37 for attrname, value in attrs:
40 if attrname == 'http-equiv':
41 http_equiv = value.lower()
42 elif attrname == 'content':
44 elif (attrname == 'charset') and (not self.charset):
45 self.charset = value.lower()
48 if (not self.charset) and (http_equiv == "content-type"):
50 # extract charset from "text/html; foo; charset=UTF-8, bar; baz;"
51 self.charset = content.lower().split('charset=')[1].split(';')[0].split(',')[0]
52 self.meta_charset = 1 # Remember that the charset was retrieved from
53 # META tag, not from the Content-Type header
57 if http_equiv == "refresh":
58 self.refresh = content
60 def start_title(self, attrs):
64 if not self.title: # use only the first title
65 self.title = self.accumulator
67 def do_link(self, attrs):
71 for attrname, value in attrs:
74 if (attrname == 'rel') and (value.lower() in ('icon', 'shortcut icon')):
76 elif attrname == 'href':
83 def parse_html(html_text, charset=None, log=None):
84 parser = HTMLParser(charset)
87 parser.feed(html_text)
88 except (HTMLParseError, HTMLHeadDone):
93 except (HTMLParseError, HTMLHeadDone):
96 if (parser.title is None) and (parser.refresh is None) and (parser.icon is None):