From a147d51d168748fe91f9ee8e27fcc065d12658d8 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Sun, 16 Dec 2007 17:24:07 +0000 Subject: [PATCH] Split parse_html.py into parse_html_htmlparser.py. git-svn-id: file:///home/phd/archive/SVN/bookmarks_db/trunk@104 fdd5c36f-1aea-0310-aeeb-c58d7e2b6c23 --- Robots/parse_html.py | 79 ++-------------------------- Robots/parse_html_htmlparser.py | 93 +++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 75 deletions(-) create mode 100644 Robots/parse_html_htmlparser.py diff --git a/Robots/parse_html.py b/Robots/parse_html.py index b30c664..8e5ca2b 100755 --- a/Robots/parse_html.py +++ b/Robots/parse_html.py @@ -1,6 +1,6 @@ #! /usr/bin/env python """ - HTML Parser + HTML Parsers wrapper Written by BroytMann. Copyright (C) 1997-2007 PhiloSoft Design """ @@ -11,14 +11,10 @@ from m_lib.defenc import default_encoding current_charset = default_encoding.replace("windows-", "cp") DEFAULT_CHARSET = "cp1251" # Stupid default for Russian Cyrillic -from HTMLParser import HTMLParseError -from m_lib.net.www.html import HTMLParser as _HTMLParser +from parse_html_htmlparser import parse_html as _parse_html -class HTMLHeadDone(Exception): pass - - -class HTMLParser(_HTMLParser): +class HTMLParser(object): def __init__(self, charset=None): _HTMLParser.__init__(self) self.charset = charset @@ -27,58 +23,6 @@ class HTMLParser(_HTMLParser): self.refresh = '' self.icon = None - def end_head(self): - raise HTMLHeadDone() - - - def do_meta(self, attrs): - http_equiv = "" - content = "" - - for attrname, value in attrs: - if value: - value = value.strip() - if attrname == 'http-equiv': - http_equiv = value.lower() - elif attrname == 'content': - content = value - - if (not self.charset) and (http_equiv == "content-type"): - try: - # extract charset from "text/html; foo; charset=UTF-8; bar;" - self.charset = content.lower().split('charset=')[1].split(';')[0] - self.meta_charset = 1 # Remember that the charset was retrieved from - # META tag, not from the Content-Type header - except IndexError: - pass - - if http_equiv == "refresh": - self.refresh = content - - - def start_title(self, attrs): - self.accumulator = '' - - def end_title(self): - if not self.title: # use only the first title - self.title = self.accumulator - - - def do_link(self, attrs): - has_icon = False - href = None - - for attrname, value in attrs: - if value: - value = value.strip().lower() - if (attrname == 'rel') and (value.lower() in ('icon', 'shortcut icon')): - has_icon = True - elif attrname == 'href': - href = value - - if has_icon: - self.icon = href - import re entity_re = re.compile("(&#[0-9]+;)") @@ -100,22 +44,7 @@ def parse_html(filename, charset=None, log=None): except (ValueError, LookupError): charset = None # ...try charset from HTML - infile = open(filename, 'r') - parser = HTMLParser(charset) - - for line in infile: - try: - parser.feed(line) - except (HTMLParseError, HTMLHeadDone): - break - - infile.close() - - try: - parser.close() - except (HTMLParseError, HTMLHeadDone): - pass - + parser = _parse_html(filename, charset) title = parser.title if not parser.charset: diff --git a/Robots/parse_html_htmlparser.py b/Robots/parse_html_htmlparser.py new file mode 100644 index 0000000..e1a35f1 --- /dev/null +++ b/Robots/parse_html_htmlparser.py @@ -0,0 +1,93 @@ +""" + HTML Parser + + Written by BroytMann. Copyright (C) 1997-2007 PhiloSoft Design +""" + +from HTMLParser import HTMLParseError +from m_lib.net.www.html import HTMLParser as _HTMLParser + + +class HTMLHeadDone(Exception): pass + + +class HTMLParser(_HTMLParser): + def __init__(self, charset=None): + _HTMLParser.__init__(self) + self.charset = charset + self.meta_charset = 0 + self.title = '' + self.refresh = '' + self.icon = None + + def end_head(self): + raise HTMLHeadDone() + + + def do_meta(self, attrs): + http_equiv = "" + content = "" + + for attrname, value in attrs: + if value: + value = value.strip() + if attrname == 'http-equiv': + http_equiv = value.lower() + elif attrname == 'content': + content = value + + if (not self.charset) and (http_equiv == "content-type"): + try: + # extract charset from "text/html; foo; charset=UTF-8; bar;" + self.charset = content.lower().split('charset=')[1].split(';')[0] + self.meta_charset = 1 # Remember that the charset was retrieved from + # META tag, not from the Content-Type header + except IndexError: + pass + + if http_equiv == "refresh": + self.refresh = content + + + def start_title(self, attrs): + self.accumulator = '' + + def end_title(self): + if not self.title: # use only the first title + self.title = self.accumulator + + + def do_link(self, attrs): + has_icon = False + href = None + + for attrname, value in attrs: + if value: + value = value.strip().lower() + if (attrname == 'rel') and (value.lower() in ('icon', 'shortcut icon')): + has_icon = True + elif attrname == 'href': + href = value + + if has_icon: + self.icon = href + + +def parse_html(filename, charset=None): + infile = open(filename, 'r') + parser = HTMLParser(charset) + + for line in infile: + try: + parser.feed(line) + except (HTMLParseError, HTMLHeadDone): + break + + infile.close() + + try: + parser.close() + except (HTMLParseError, HTMLHeadDone): + pass + + return parser -- 2.39.5