-#! /usr/local/bin/python -O
+#! /usr/bin/env python
"""
HTML Parser
- Written by BroytMann, Jun 2002 - Aug 2002. Copyright (C) 1997-2002 PhiloSoft Design
+ Written by BroytMann. Copyright (C) 1997-2007 PhiloSoft Design
"""
import sys
current_charset = sys.getdefaultencoding()
-DEFAULT_CHARSET = "windows-1251"
+if current_charset == "ascii":
+ try:
+ import locale
+ except ImportError:
+ pass
+ else:
+ current_charset = locale.getpreferredencoding()
+current_charset = current_charset.replace("windows-", "cp").lower()
+DEFAULT_CHARSET = "cp1251" # Stupid default for Russian Cyrillic
from HTMLParser import HTMLParseError
-from m_lib.www.html import HTMLParser as _HTMLParser
+from m_lib.net.www.html import HTMLParser as _HTMLParser
class HTMLHeadDone(Exception): pass
try:
# extract charset from "text/html; foo; charset=UTF-8; bar;"
self.charset = content.lower().split('charset=')[1].split(';')[0]
- self.meta_charset = 1
+ self.meta_charset = 1 # Remember that the charset was retrieved from
+ # META tag, not from the Content-Type header
except IndexError:
pass
def start_title(self, attrs):
self.accumulator = ''
+
def end_title(self):
if not self.title: # use only the first title
self.title = self.accumulator
+ def do_link(self, attrs):
+ has_icon = False
+ href = None
+
+ for attrname, value in attrs:
+ if value:
+ value = value.strip().lower()
+ if (attrname == 'rel') and (value in ('icon', 'shortcut icon')):
+ has_icon = True
+ elif attrname == 'href':
+ href = value
+
+ if has_icon:
+ self.icon = href
+ else:
+ self.icon = None
+
+
+import re
+entity_re = re.compile("(&#[0-9]+;)")
+
+def recode_entities(title, charset):
+ output = []
+ for part in entity_re.split(title):
+ if entity_re.match(part):
+ part = unichr(int(part[2:-1])).encode(charset, "replace")
+ output.append(part)
+
+ return ''.join(output)
+
+
def parse_html(filename, charset=None, log=None):
infile = open(filename, 'r')
parser = HTMLParser(charset)
except (HTMLParseError, HTMLHeadDone):
pass
+ title = parser.title
+
if not parser.charset:
- title = parser.title
- ascii = 1
- for c in title:
- if not (32 <= ord(c) <= 127): # non-ASCII character
- ascii = 0
- break
- if not ascii:
+ try:
+ unicode(title, "ascii")
+ except UnicodeDecodeError:
parser.charset = DEFAULT_CHARSET
+
+ if parser.charset:
+ parser.charset = parser.charset.replace("windows-", "cp").lower()
+
if parser.charset and (parser.charset <> current_charset):
try:
if parser.meta_charset:
if log: log(" META charset : %s" % parser.charset)
else:
if log: log(" charset : %s" % parser.charset)
- if log: log(" title : %s" % parser.title)
- parser.title = unicode(parser.title, parser.charset, "replace").encode(current_charset, "replace")
+ if log: log(" title : %s" % title)
+ title = unicode(title, parser.charset, "replace").encode(current_charset, "replace")
if log: log(" current charset: %s" % current_charset)
- if log: log(" converted title: %s" % parser.title)
+ if log: log(" converted title: %s" % title)
except LookupError:
if log: log(" unknown charset: `%s' or `%s'" % (parser.charset, current_charset))
+ parser.title = recode_entities(title, current_charset)
return parser
print parser.charset
print parser.title
print parser.refresh
+ print parser.icon