From 896c8750d5b4e7a833156cc87a1f2f5847729904 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Tue, 4 Mar 2008 10:32:18 +0000 Subject: [PATCH] Reparse the HTML if the charset was changed. git-svn-id: file:///home/phd/archive/SVN/bookmarks_db/trunk@196 fdd5c36f-1aea-0310-aeeb-c58d7e2b6c23 --- Robots/parse_html_beautifulsoup.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Robots/parse_html_beautifulsoup.py b/Robots/parse_html_beautifulsoup.py index 632ee9f..93a4d6f 100644 --- a/Robots/parse_html_beautifulsoup.py +++ b/Robots/parse_html_beautifulsoup.py @@ -49,15 +49,23 @@ class BadDeclParser(BeautifulSoup): return j -def parse_html(filename, charset=None, log=None): +def _parse_html(filename, charset): infile = open(filename, 'r') try: - root = BadDeclParser(infile, fromEncoding=charset) + return BadDeclParser(infile, fromEncoding=charset) except TypeError: return None finally: infile.close() +def parse_html(filename, charset=None, log=None): + root = _parse_html(filename, charset) + + _charset = root.originalEncoding + if _charset in ("ISO-8859-2", "windows-1252", "MacCyrillic"): # Replace default + _charset = DEFAULT_CHARSET + root = _parse_html(filename, _charset) + try: html = root.html except AttributeError: @@ -76,10 +84,6 @@ def parse_html(filename, charset=None, log=None): if head is None: head = html # Some sites put TITLE in HTML without HEAD - _charset = root.originalEncoding - if _charset in ("ISO-8859-2", "windows-1252", "MacCyrillic"): # Replace default - _charset = DEFAULT_CHARSET - try: title = head.title.string.encode(_charset) except AttributeError: -- 2.39.5