From daa40d9fc61a15fe51955d97b5711af5973a64d3 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Tue, 18 Dec 2007 05:52:48 +0000 Subject: [PATCH] Recode from DEFAULT_CHARSET if recoding from cp1252 (bad guess by BeautifulSoup) failed. git-svn-id: file:///home/phd/archive/SVN/bookmarks_db/trunk@112 fdd5c36f-1aea-0310-aeeb-c58d7e2b6c23 --- Robots/parse_html.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Robots/parse_html.py b/Robots/parse_html.py index 3e20a55..3250a0d 100755 --- a/Robots/parse_html.py +++ b/Robots/parse_html.py @@ -54,10 +54,20 @@ def parse_html(filename, charset=None, log=None): if parser.meta_charset: if log: log(" META charset : %s" % parser.charset) else: - if log: log(" charset : %s" % parser.charset) - if log: log(" title : %s" % title) - title = unicode(title, parser.charset, "replace").encode(current_charset, "replace") + if log: log(" HTTP charset : %s" % parser.charset) if log: log(" current charset: %s" % current_charset) + if log: log(" title : %s" % title) + save_title = title + try: + title = unicode(title, parser.charset).encode(current_charset) + except UnicodeError: + if parser.meta_charset and parser.charset.endswith("1252") and \ + not DEFAULT_CHARSET.endswith("1252") and (DEFAULT_CHARSET <> current_charset): + parser.charset = DEFAULT_CHARSET + if log: log(" incorrect conversion from cp1252, converting from %s" % DEFAULT_CHARSET) + title = unicode(save_title, DEFAULT_CHARSET, "replace").encode(current_charset, "replace") + else: + title = unicode(title, parser.charset, "replace").encode(current_charset, "replace") if log: log(" converted title: %s" % title) except LookupError: if log: log(" unknown charset: `%s' or `%s'" % (parser.charset, current_charset)) -- 2.39.5