From f84aa81f636d43180118cd3e9470d3d57f9c539e Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Fri, 28 Dec 2007 16:39:48 +0000 Subject: [PATCH] Do not display too much titles if they are equal. git-svn-id: file:///home/phd/archive/SVN/bookmarks_db/trunk@145 fdd5c36f-1aea-0310-aeeb-c58d7e2b6c23 --- Robots/parse_html.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Robots/parse_html.py b/Robots/parse_html.py index d3870fa..9f924c4 100755 --- a/Robots/parse_html.py +++ b/Robots/parse_html.py @@ -53,7 +53,7 @@ def parse_html(filename, charset=None, log=None): else: if log: log("Parser %s.%s failed, trying next one." % (p.__module__, p.__name__)) - title = parser.title + converted_title = title = parser.title if not parser.charset: try: unicode(title, "ascii") @@ -73,24 +73,26 @@ def parse_html(filename, charset=None, log=None): if log: log(" title : %s" % title) save_title = title try: - title = unicode(title, parser.charset).encode(current_charset) + converted_title = unicode(title, parser.charset).encode(current_charset) except UnicodeError: if parser.meta_charset and parser.charset.endswith("1252") and \ not DEFAULT_CHARSET.endswith("1252") and (DEFAULT_CHARSET <> current_charset): parser.charset = DEFAULT_CHARSET if log: log(" incorrect conversion from cp1252, converting from %s" % DEFAULT_CHARSET) - title = unicode(save_title, DEFAULT_CHARSET, "replace").encode(current_charset, "replace") + converted_title = unicode(save_title, DEFAULT_CHARSET, "replace").encode(current_charset, "replace") else: - title = unicode(title, parser.charset, "replace").encode(current_charset, "replace") - if log: log(" converted title: %s" % title) + converted_title = unicode(title, parser.charset, "replace").encode(current_charset, "replace") + if log and (converted_title <> title): log(" converted title: %s" % converted_title) except LookupError: if log: log(" unknown charset: `%s' or `%s'" % (parser.charset, current_charset)) - - title = recode_entities(title, current_charset) - parts = [s.strip() for s in title.replace('\r', '').split('\n')] - title = ' '.join([s for s in parts if s]) - if log: log(" final title : %s" % title) - parser.title = title + else: + if log: log(" title : %s" % title) + + final_title = recode_entities(converted_title, current_charset) + parts = [s.strip() for s in final_title.replace('\r', '').split('\n')] + final_title = ' '.join([s for s in parts if s]) + if log and (final_title <> converted_title): log(" final title : %s" % final_title) + parser.title = final_title return parser -- 2.39.5