From: Oleg Broytman Date: Tue, 18 Dec 2007 05:55:31 +0000 (+0000) Subject: Try BeautifulSoup; if it fails - fall back to HTML Parser. X-Git-Tag: v4.5.3~263 X-Git-Url: https://git.phdru.name/?a=commitdiff_plain;h=b823c1b58711d6c4ed96fcb31de5cf348d8a6c1a;p=bookmarks_db.git Try BeautifulSoup; if it fails - fall back to HTML Parser. git-svn-id: file:///home/phd/archive/SVN/bookmarks_db/trunk@113 fdd5c36f-1aea-0310-aeeb-c58d7e2b6c23 --- diff --git a/Robots/parse_html.py b/Robots/parse_html.py index 3250a0d..fc9514d 100755 --- a/Robots/parse_html.py +++ b/Robots/parse_html.py @@ -11,10 +11,16 @@ from m_lib.defenc import default_encoding current_charset = default_encoding.replace("windows-", "cp") DEFAULT_CHARSET = "cp1251" # Stupid default for Russian Cyrillic +parsers = [] try: - from parse_html_beautifulsoup import parse_html as _parse_html + from parse_html_beautifulsoup import parse_html except ImportError: - from parse_html_htmlparser import parse_html as _parse_html + pass +else: + parsers.append(parse_html) + +from parse_html_htmlparser import parse_html +parsers.append(parse_html) import re @@ -37,9 +43,14 @@ def parse_html(filename, charset=None, log=None): except (ValueError, LookupError): charset = None # ...try charset from HTML - parser = _parse_html(filename, charset) - title = parser.title + for p in parsers: + parser = p(filename, charset) + if parser: + break + else: + if log: log("Parser %s failed, trying next one." % p) + title = parser.title if not parser.charset: try: unicode(title, "ascii") diff --git a/Robots/parse_html_beautifulsoup.py b/Robots/parse_html_beautifulsoup.py index 8c51d05..3408def 100644 --- a/Robots/parse_html_beautifulsoup.py +++ b/Robots/parse_html_beautifulsoup.py @@ -20,34 +20,29 @@ class BSoupParser(HTMLParser): def parse_html(filename, charset=None): infile = open(filename, 'r') - root = BeautifulSoup(infile, fromEncoding=charset) + try: + root = BeautifulSoup(infile, fromEncoding=charset) + except TypeError: + return None infile.close() _charset = root.originalEncoding try: title = root.html.head.title.string.encode(_charset) except AttributeError: - title = '' + return None - try: - meta = root.html.head.find(_find_refresh, recursive=False) - except AttributeError: - refresh = None + meta = root.html.head.find(_find_refresh, recursive=False) + if meta: + refresh = meta.get("content") else: - if meta: - refresh = meta.get("content") - else: - refresh = None + refresh = None - try: - meta = root.html.head.find(_find_icon, recursive=False) - except AttributeError: - icon = None + meta = root.html.head.find(_find_icon, recursive=False) + if meta: + icon = meta.get("href") else: - if meta: - icon = meta.get("href") - else: - icon = None + icon = None return BSoupParser(_charset, _charset != charset, title, refresh, icon)