-#! /usr/local/bin/python -O
+#! /usr/bin/env python
"""
- HTML Parser
+ HTML Parsers wrapper
- Written by BroytMann, Jun 2002 - May 2003. Copyright (C) 1997-2003 PhiloSoft Design
+ Written by BroytMann. Copyright (C) 1997-2007 PhiloSoft Design
"""
+import codecs
-import sys
-current_charset = sys.getdefaultencoding()
-DEFAULT_CHARSET = "windows-1251" # Stupid default for Russian Cyrillic
+from m_lib.defenc import default_encoding
+current_charset = default_encoding.replace("windows-", "cp")
+DEFAULT_CHARSET = "cp1251" # Stupid default for Russian Cyrillic
-
-from HTMLParser import HTMLParseError
-from m_lib.www.html import HTMLParser as _HTMLParser
-
-
-class HTMLHeadDone(Exception): pass
-
-
-class HTMLParser(_HTMLParser):
- def __init__(self, charset=None):
- _HTMLParser.__init__(self)
- self.charset = charset
- self.meta_charset = 0
- self.title = ''
- self.refresh = ''
-
- def end_head(self):
- raise HTMLHeadDone()
-
-
- def do_meta(self, attrs):
- http_equiv = ""
- content = ""
-
- for attrname, value in attrs:
- if value:
- value = value.strip()
- if attrname == 'http-equiv':
- http_equiv = value.lower()
- elif attrname == 'content':
- content = value
-
- if (not self.charset) and (http_equiv == "content-type"):
- try:
- # extract charset from "text/html; foo; charset=UTF-8; bar;"
- self.charset = content.lower().split('charset=')[1].split(';')[0]
- self.meta_charset = 1 # Remember that the charset was retrieved from
- # META tag, not from the Content-Type header
- except IndexError:
- pass
-
- if http_equiv == "refresh":
- self.refresh = content
-
-
- def start_title(self, attrs):
- self.accumulator = ''
-
- def end_title(self):
- if not self.title: # use only the first title
- self.title = self.accumulator
+try:
+ from parse_html_beautifulsoup import parse_html as _parse_html
+except ImportError:
+ from parse_html_htmlparser import parse_html as _parse_html
import re
def parse_html(filename, charset=None, log=None):
- infile = open(filename, 'r')
- parser = HTMLParser(charset)
-
- for line in infile:
+ if charset:
try:
- parser.feed(line)
- except (HTMLParseError, HTMLHeadDone):
- break
-
- infile.close()
-
- try:
- parser.close()
- except (HTMLParseError, HTMLHeadDone):
- pass
+ codecs.lookup(charset) # In case of unknown charset...
+ except (ValueError, LookupError):
+ charset = None # ...try charset from HTML
+ parser = _parse_html(filename, charset)
title = parser.title
if not parser.charset:
- ascii = 1
- for c in title:
- if not (32 <= ord(c) <= 127): # non-ASCII character
- ascii = 0
- break
- if not ascii:
+ try:
+ unicode(title, "ascii")
+ except UnicodeDecodeError:
parser.charset = DEFAULT_CHARSET
+ if parser.charset:
+ parser.charset = parser.charset.replace("windows-", "cp").lower()
+
if parser.charset and (parser.charset <> current_charset):
try:
if parser.meta_charset:
if __name__ == '__main__':
import sys
- parser = parse_html(sys.argv[1])
+ parser = parse_html(sys.argv[1], current_charset)
print parser.charset
print parser.title
print parser.refresh
+ print parser.icon