From 240c6b89a3825d71e040702dc9f9f948a71da107 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Wed, 13 Feb 2008 18:18:18 +0000 Subject: [PATCH] Stop meddling with cp1252. git-svn-id: file:///home/phd/archive/SVN/bookmarks_db/trunk@177 fdd5c36f-1aea-0310-aeeb-c58d7e2b6c23 --- Robots/parse_html.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/Robots/parse_html.py b/Robots/parse_html.py index bc50767..c7acb64 100755 --- a/Robots/parse_html.py +++ b/Robots/parse_html.py @@ -12,11 +12,12 @@ DEFAULT_CHARSET = "cp1251" # Stupid default for Russian Cyrillic parsers = [] try: - from parse_html_beautifulsoup import parse_html + import parse_html_beautifulsoup + parse_html_beautifulsoup.DEFAULT_CHARSET = DEFAULT_CHARSET except ImportError: pass else: - parsers.append(parse_html) + parsers.append(parse_html_beautifulsoup.parse_html) from parse_html_htmlparser import parse_html parsers.append(parse_html) @@ -79,17 +80,12 @@ def parse_html(filename, charset=None, log=None): if log: log(" HTTP charset : %s" % parser.charset) if log: log(" current charset: %s" % universal_charset) if log: log(" title : %s" % title) - save_title = title try: converted_title = unicode(title, parser.charset).encode(universal_charset) except UnicodeError: - if parser.meta_charset and parser.charset.endswith("1252") and \ - not DEFAULT_CHARSET.endswith("1252") and (DEFAULT_CHARSET <> universal_charset): - parser.charset = DEFAULT_CHARSET - if log: log(" incorrect conversion from cp1252, converting from %s" % DEFAULT_CHARSET) - converted_title = unicode(save_title, DEFAULT_CHARSET, "replace").encode(universal_charset, "replace") - else: - converted_title = unicode(title, parser.charset, "replace").encode(universal_charset, "replace") + if log: log(" incorrect conversion from %s, converting from %s" % (parser.charset, DEFAULT_CHARSET)) + converted_title = unicode(title, DEFAULT_CHARSET, "replace").encode(universal_charset, "replace") + parser.charset = DEFAULT_CHARSET if log and (converted_title <> title): log(" converted title: %s" % converted_title) except LookupError: if log: log(" unknown charset: `%s' or `%s'" % (parser.charset, universal_charset)) -- 2.39.2